
    5i~                     
   d Z ddlmZmZmZmZ ddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlmZ ddlZddlZddlmZ ddlZddlZddlZddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'Z'ddl(Z)ddejT                  fdZ+dejT                  ddfdZ,ejT                  dddddfdZ-d Z.d Z/ddZ0d Z1ddZ2ejT                  dddddfdZ3	 	 	 d dZ4d!dZ5d"dZ6ddejT                  dfdZ7y)#z|
Helper Functions
****************

The following helper functions are useful shortcuts for interacting with File objects.

    )print_functionunicode_literalsdivisionabsolute_importN)defaultdict)	checksums   )logger   )dxfileDXFile)FILE_REQUEST_TIMEOUT)DXErrorDXFileErrorDXPartLengthMismatchErrorDXChecksumMismatchErrorDXIncompleteReadsErrorerr_exit)
md5_hasher)response_iteratorc                      t        | |||      S )a}  
    :param dxid: file ID
    :type dxid: string
    :rtype: :class:`~dxpy.bindings.dxfile.DXFile`

    Given the object ID of an uploaded file, returns a remote file
    handler that is a Python file-like object.

    Example::

      with open_dxfile("file-xxxx") as fd:
          for line in fd:
              ...

    Note that this is shorthand for::

      DXFile(dxid)

    )projectmoderead_buffer_size)r   )dxidr   r   r   s       /home/marpiech/ifpan-abm-pgxpred/analysis/marpiech-gwas-test/venv/lib/python3.12/site-packages/dxpy/bindings/dxfile_functions.pyopen_dxfiler   0   s    ( $dEUVV    Fc                 H    t        | |||      } |j                  di | |S )a|  
    :param mode: One of "w" or "a" for write and append modes, respectively
    :type mode: string
    :rtype: :class:`~dxpy.bindings.dxfile.DXFile`

    Additional optional parameters not listed: all those under
    :func:`dxpy.bindings.DXDataObject.new`.

    Creates a new remote file object that is ready to be written to;
    returns a :class:`~dxpy.bindings.dxfile.DXFile` object that is a
    writable file-like object.

    Example::

        with new_dxfile(media_type="application/json") as fd:
            fd.write("foo\n")

    Note that this is shorthand for::

        dxFile = DXFile()
        dxFile.new(**kwargs)

    )r   write_buffer_sizeexpected_file_sizefile_is_mmapd )r   new)r   r    r!   r"   kwargsdx_files         r   
new_dxfiler'   G   s/    2 $2CXj#02GGKK&Nr      c           
      V    t        d       }	d}
|
st        | ||	f||||||d|}
|
syy)a  
    :param dxid: DNAnexus file ID or DXFile (file handler) object
    :type dxid: string or DXFile
    :param filename: Local filename
    :type filename: string
    :param append: If True, appends to the local file (default is to truncate local file if it exists)
    :type append: boolean
    :param project: project to use as context for this download (may affect
            which billing account is billed for this download). If None or
            DXFile.NO_PROJECT_HINT, no project hint is supplied to the API server.
    :type project: str or None
    :param describe_output: (experimental) output of the file-xxxx/describe API call,
            if available. It will make it possible to skip another describe API call.
            It should contain the default fields of the describe API call output and
            the "parts" field, not included in the output by default.
    :type describe_output: dict or None
    :param symlink_max_tries: Maximum amount of tries when downloading a symlink with aria2c.
    :type symlink_max_tries: int or None

    Downloads the remote file referenced by *dxid* and saves it to *filename*.

    Example::

        download_dxfile("file-xxxx", "localfilename.fastq")

    c                       y)N   r#   r#   r   r   <lambda>z!download_dxfile.<locals>.<lambda>   s    r   F)	chunksizeappendshow_progressr   describe_outputsymlink_max_triesN)r   _download_dxfile)r   filenamer-   r.   r/   r   r0   r1   r%   part_retry_countersuccesss              r   download_dxfiler6   f   sQ    : %Y/G"4#+#5	- .7*01>+23B5F	- &,	- r   c                     d }t         j                  d   j                  t         j                        D ].  }t         j                  j                  ||       } ||      s,|c S  y )Nc                     t         j                  j                  |       xr$ t        j                  | t         j                        S N)ospathisfileaccessX_OK)fpaths    r   is_exez_which.<locals>.is_exe   s)    ww~~e$B5"'')BBr   PATH)r:   environsplitpathsepr;   join)programr@   r;   exe_files       r   _whichrH      sW    C 

6"((4 77<<g.(O r   c                    t        d      }|t        d       |d| g}	 t        d       t        j                  |      }j                         j                         }t        |      dk7  rt        dt        |      z          |d   }|j                  d	      }||k7  r&t        d
t        |      z   dz   t        |      z          t        d       y # t        j
                  $ r t        dt        |      z          Y w xY w)Nmd5sumz&md5sum is not installed on this systemz-bzCalculating checksumzFailed to run md5sum: r	   zmd5sum returned weird results: r   asciizChecksum doesn't match z  expected:zChecksum correct)rH   r   print
subprocesscheck_outputCalledProcessErrorstrstriprC   lenencode)r3   	md5digest
md5sum_execmdcmd_outline
actual_md5s          r   _verifyrZ      s    !J9:tX
&C6$%))#. ==?  "D
4yA~2SY>?aJ  )I Y*S_<}LsS\~]^	
 (( 6)CH456s    C *C<;C<c                 h   |dk  rt         j                  j                  d      t        d      }|t	        d       y t        | t              r| }nt        j                  |       }|j                  dd|      \  }}t        dt        j                               }	dd	d
t        |	      dt        |	      dg}
|
j                  dt        |      g       t        j                         }t        j                  j!                  |      \  }}|d|fv r|n|}|
d|dt        j                  j#                  |      |gz  }
	 t%        j&                  |
t$        j(                         |t/        ||       y y # t$        j*                  $ r/}t	        dj-                  t        |
      |             Y d }~Md }~ww xY w)Nr   z5symlink_max_tries argument has to be positive integeraria2czxaria2c must be installed on this system to download this data. Please see the documentation at https://aria2.github.io/.Ti`T  )preauthenticateddurationr      z-cz-sz-xz--retry-wait=10z-m z-oz-d)stderrz%Failed to call download: {cmd}
{msg}
)rV   msg)dxpy
exceptionsr   rH   r   
isinstancer   get_download_urlminmultiprocessing	cpu_countrP   extendr:   getcwdr;   rC   abspathrM   
check_callSTDOUTrO   formatrZ   )r   rT   r   dest_filenamer1   
aria2c_exedxfurl_headersmax_connectionsrV   cwd	directoryr3   es                  r   _download_symbolic_linkry      s   1oo%%&]^^ !J M 	N$kk$(($5;4; ) =MC
 "o779:Oc/"c/"C JJc+,-. ))+C''--6Ix RI-9ID(D"''//)"<cBBCXc**;*;< y)  (( X:AAc#hTUAVWWXs   :%E/ /F1%F,,F1c           	         |y | j                  |      }|t        dj                  ||            |j                  d      }d d d d d d}||vrt        d	j                  |            |t        d
j                  ||            t        j                  |      } ||   |      }||k7  rt        dj                  |||||            y)NzPart {} not found in {}checksumc                 L    t        j                  |       j                  dd      S N   big)zlibcrc32to_bytesdatas    r   r,   z"_verify_checksum.<locals>.<lambda>   s    djj.775A r   c                 L    t        j                   |       j                  dd      S r}   )crc32cr   r   s    r   r,   z"_verify_checksum.<locals>.<lambda>   s    v}}T2;;AuE r   c                 H    t        j                  |       j                         S r9   )hashlibsha1digestr   s    r   r,   z"_verify_checksum.<locals>.<lambda>   s    W\\$/668 r   c                 H    t        j                  |       j                         S r9   )r   sha256r   r   s    r   r,   z"_verify_checksum.<locals>.<lambda>   s    w~~d3::< r   c                 L    t        j                  |       j                  dd      S )N   r   )r   	crc64nvmer   r   s    r   r,   z"_verify_checksum.<locals>.<lambda>   s    )"5"5d";"D"DQ"N r   )CRC32CRC32CSHA1SHA256	CRC64NVMEzUnsupported checksum type: {}z {} checksum not found in part {}z;{} checksum mismatch in {} in part {} (expected {}, got {})T)getr   ro   base64	b64decoder   )	partspart_id
chunk_datachecksum_type	dxfile_idpartexpected_checksum	verifiersgot_checksums	            r   _verify_checksumr      s	   99WD|3::7INOO,AE8<NI I%9@@OPP <CCMSZ[\\(():;+9]+J7L((%&c&j&jkx  {D  FM  O`  bn  'o  p  	pr   c                 x   d}|xs d}| |kD  r| }t        t        | t        |      z  |z              }t        t        j                  | t        |      z  dz              }d}t
        j                  j                  d       t
        j                  j                  |j                  ||dkD  rd|dz
  z  dz   nd	d
||z
  z  | |rdj                  |      nd	||             t
        j                  j                          t
        j                  j                  d       t
        j                  j                          y )N<   r   d   zN[{done}{pending}] {action} {done_bytes:,}{remaining} bytes ({percent}%) {name}z[2Kr   =>r`    z of {size:,})size)actiondonepending
done_bytes	remainingpercentname)
introundfloatmathfloorsysra   writero   flush)	bytes_downloaded	file_sizer3   r   	num_tickseffective_file_sizeticksr   fmts	            r   _print_progressr     s   I#.q--.'%0C*DD	QRSE$**.7J1KKsRSTG
ZC JJXJJSZZvBG!)cUQY&7#&=QS(+y5/@(A+;S\.*?*?Y*?*Obd(/%-   / 0 JJJJTJJr   c	           
      
  	"#$% d}
t        | t              r| "n#t        | dt        j                  k7  rnd      "|r|j                  d      |}n "j                  d)dhdd	}d|v rd|vsd|v r%|d   d	k(  rd
|v r|d
   }nd}t        | |||       y|d   $t        $t              %|j                  d      }|j                  d      }d}%D ]  }|$|   d<   |$|   d   z  } |rt        |d      }n	 t        |d      }|rt        dd|       d*"	$fd	#d*#$%fd	}d*"$fd	}|5  d}|j                  dk(  rd\  }}	 %D ]@  }$|   }d
|vr)t        dj                   "j                                     |d   }t               }|dkD  ri|j!                  t#        ||            }t%        |      t#        ||      k  rt        dj                  |            |j'                  |       ||z  }|dkD  ri|j)                         |d
   k7  rt        dj                  |            |j                  d       t+        $|| "j                                |}|j-                         }|s*|
|d   z  }
t        |
||d       C 	 |j3                  |       |j5                          |%d%j7                  |      dz   = |r&t%        %      t%        $      k  rt        |||d       t/        j0                  d|t%        %             	 |dkD  xr |dk(  xr t8        j:                  }d \  }}}d}|r%|j                  d!      r|d!   j                  d"      }t=         ||      "j>                  |#      D ]  \  } }!| |k7  rH |||||       | dt               }}}|j                  d      t+        $||!| "j                                |t%        |!      z  }|j'                  |!       |jA                  |!       |s|
t%        |!      z  }
t        |
||         |||||       |rt        |
||d$       |rtH        jJ                  jA                  d(       	 ddd       y# t        $ r t        |d      }Y zw xY w# t        t        f$ r }t/        j0                  |       Y d}~d}~ww xY w# t        $ r tC        tE        jF                         tH        jJ                  %       |xx   dz  cc<   ||   dkD  rHtC        d&j                   "j                         ||   |      tH        jJ                  %       Y ddd       y' w xY w# 1 sw Y   yxY w)+a4  
    Core of download logic. Download file-id *dxid* and store it in
    a local file *filename*.

    The return value is as follows:
    - True means the download was successfully completed
    - False means the download was stopped because of a retryable error
    - Exception raised for other errors
    r   rN)r   r   r   T)fieldsdefault_fieldsdrivezdrive-PUBLISHEDmd5)r1   )keyr   checksumTypestartabzrb+wbc                      j                   ddi	\  }}|||d<   d}t        
      dkD  s|dkD  s||z
  dz   
|    d   k  rd}t        j                  ||||t        |      }| |fS )	Nr   zIf-MatchFr   r   r   Tr#   )rf   rR   rc   _dxhttp_read_ranger   )part_id_to_getr   ende_tagrs   headers	sub_ranger   r   r%   r   r   s           r   	get_chunkz#_download_dxfile.<locals>.get_chunkX  s    .v..IwI&IW"'GJ 	u:>eaiS5[1_u^?TU[?\-\I&&sGUCAUW`at##r   c              3      K   D ]K  }|   }t        |d   |d   |d   z         D ](  }t        |z   |d   |d   z         dz
  }|||| gi f * M y w)Nr   r   r   )rangerg   )	r   part_id_to_chunk	part_infochunk_start	chunk_endr-   r   r   parts_to_gets	        r   chunk_requestsz(_download_dxfile.<locals>.chunk_requestse  s      , 	W./I$Yw%779KiX^N_9_ajk Wi 779KiX^N_9_`cdd	"2KE!RTVVVW	Ws   AAc                    |
d|    vr|y |A||    d   k7  r6d}|j                   j                         | |    d   |      }t        |      |;d|    vr4t        j                  dj                   j                                      y |^|j                         |    d   k7  rDd}|j                   j                         | |    d   |j                               }t        |      y y )Nr   r   z=Unexpected part data size in {} part {} (expected {}, got {})z6Download of file {} is not being checked for integrityz3md5sum mismatch in {} part {} (expected {}, got {}))ro   get_idr   warningswarn	hexdigestr   )_part_id	got_byteshasherr   rb   r   r   s        r   verify_partz%_download_dxfile.<locals>.verify_partl  s    U%/%AeFW Y%/&2I%IQC**]V]]_hh8OQZ[C+C00%uX">MMRYYZgZ`ZgZgZijkF$4$4$6%/%:P$PGC**]V]]_hh8NPVP`P`PbcC)#.. %Qr   )Ni   z+File {} does not contain part md5 checksumsz#Local data for part {} is truncatedz3Checksum mismatch when verifying downloaded part {}Verified)r   r   zResuming atzVerified %s/%d downloaded partsi   )NNNsymlinkTargetIdentifierETag)do_first_task_sequentially	Completed)filez)Retrying {} ({} tries remain for part {})F
r#   r9   )&re   r   NO_PROJECT_HINTr   describery   sortedr   openIOErrorr   r   r   ro   r   r   readrg   rR   updater   r   tellr
   debugseektruncateindexrc   JOB_IDr   _http_threadpoolr   rL   	traceback
format_excr   ra   )&r   r3   r4   r-   r.   r/   r   r0   r1   r%   _bytesdxfile_descr   r   r   offsetr   fhr   r   last_verified_poslast_verified_partmax_verify_chunk_sizer   bytes_to_readr   chunkrx   get_first_chunk_sequentiallycur_partr   r   
chunk_partr   r   r   r   r   s&      `  `  `                        @@@@r   r2   r2      s    F$3GvG]G]<]cgi?..w7C%%fooVgYtVvV +'"<+%+g*>BS*SKe$CCc7HPab E%S)L'IOON3MF )"(gw%.(() (D!	&h&B 4*$ $W W/" 
 M77e 9H5 5 + \G %gII-)*W*^*^_l_e_l_l_n*opp$-f$5M'\F'!+ ",A=(Q Ru:,A=(QQ"-.S.Z.Z[b.c"dde,%)>> (!+ '')Yu-==)*_*f*fgn*opp"w/;(}v}}_-4*,.GGI)("i&77F+FIxPZ[+\0 GG%&KKM!- !J,"4"45G"H"J!JK\!2SZ!? 19h}]LL:<NPST`Pab 	,5
,B,mGX\]G],mbfbmbm(*:'HiE?#6#67P#Q'(ABFFvN*;N5<Q<B<S<SWs+u A&
J ))VUC2<aiH"w/;(*mUbU[UbUbUdeS_,	j)$ c*o-F#FIx@A )VU;	8KP JJT"[M MW  	&h%B	&R [)  Q F  	)&&(szz:x(A-(!(+a/AHHZlmuZv  yA  B::'OM MP 	CM Ms   Q/ 7UB4RA5R<RBUC0R=9R="U/RRR:R5/U5R::U=B	UUUUUc	                 t   	  nt         d      	 t        j                  j                               j                  d fd}
 fd}	fd}d}d}t        d	      t        j                  ||k  r"|d
z  }|r|}n |       }t        j                  j                  	      \  }}dd |j                  di | d|_        |r	 ||d       	  |
|j                        }t        |      z  t        |      dk(  rn |j                  |f|r|nd|d| J |j                   dd|r|ndi| |r=t"        j$                  j                  d       t"        j$                  j!                          	 |j'                           j1                          	 |s j0                  d||r|ndd S #  dY xY w# t(        $ rW |r$t+        j,                  dj/                                ||kD  r |r%t+        j,                  dj/                  ||             Y w xY w)a  
    :param filename: Local filename
    :type filename: string
    :param file: File-like object
    :type file: File-like object
    :param media_type: Internet Media Type
    :type media_type: string
    :param keep_open: If False, closes the file after uploading
    :type keep_open: boolean
    :param write_buffer_size: Buffer size to use for upload
    :type write_buffer_size: int
    :param wait_on_close: If True, waits for the file to close
    :type wait_on_close: boolean
    :param use_existing_dxfile: Instead of creating a new file object, upload to the specified file
    :type use_existing_dxfile: :class:`~dxpy.bindings.dxfile.DXFile`
    :param multithread: If True, sends multiple write requests asynchronously
    :type multithread: boolean
    :returns: Remote file handler
    :rtype: :class:`~dxpy.bindings.dxfile.DXFile`

    Additional optional parameters not listed: all those under
    :func:`dxpy.bindings.DXDataObject.new`.

    Exactly one of *filename* or *file* is required.

    Uploads *filename* or reads from *file* into a new file object (with
    media type *media_type* if given) and returns the associated remote
    file handler. The "name" property of the newly created remote file
    is set to the basename of *filename* or to *file.name* (if it
    exists).

    Examples::

      # Upload from a path
      dxpy.upload_local_file("/home/ubuntu/reads.fastq.gz")
      # Upload from a file-like object
      with open("reads.fastq") as fh:
          dxpy.upload_local_file(file=fh)

    Nrbr   c                     t        | d      syt        j                  | j                               j                  }t        j                  |      xs t        j                  |       S )NfilenoF)hasattrr:   fstatr
  st_modestatS_ISCHRS_ISFIFO)fdr   s     r   can_be_mmapdz'upload_local_file.<locals>.can_be_mmapd  sI    r8$xx		$,,LL&=$--*=>>r   c                            sj                  |       S t        z
  d      }|dk(  ryt        j                  j                         t	        | |      t        j
                        S )z
        Returns a string or mmap'd data containing the next num_bytes of
        the file, or up to the end if there are fewer than num_bytes
        left.
        r   r   )r   r=   )r   maxmmapr
  rg   ACCESS_READ)	num_bytesbytes_availabler  r  r   r   s     r   r   zupload_local_file.<locals>.read  sa     B779%%i&0!4ayyc)_&Ef]a]m]mnnr   c           
      \   | xj                   |z  c_         dkD  rt        t        | j                   t              z  z              }t        t        | j                   t              z  dz              }d}t        j
                  j                  |j                  |dkD  rd|dz
  z  dz   ndd|z
  z  | j                   |nd	             t        j
                  j                          t        j
                  j                  d
       t        j
                  j                          y y )Nr   r   zP[{done}{pending}] Uploaded {done_bytes:,} of {total:,} bytes ({percent}%) {name}r   r   r   r`   r   )r   r   r   totalr   r   r   )	_num_bytes_transmittedr   r   r   r   ra   r   ro   r   )handlerr  r   r   r   r   r3   r   s        r   report_progressz*upload_local_file.<locals>.report_progress  s   &&)3&q=w==i@PPT]]^_E%!?!?%	BR!RVY YZ[GdCJJSZZPQ	SEAI->-DWY03y57H0I3:3Q3Q.7079A9MXSU ( W X JJJJT"JJ r   c           	         j                         }dvrT| #t        j                  j                  |       |d<   n/	 j                  }t        j                  j                  |      |d<   t        ddd|S # t
        $ r Y w xY w)Nr   a)r   
media_typer    r!   r"   r#   )copyr:   r;   basenamer   AttributeErrorr'   )	r3   creation_kwargslocal_file_namer   r"   r   r%   r   r    s	      r   get_new_handlerz*upload_local_file.<locals>.get_new_handler%  s     !++-#*,''*:*:8*D'P&*iiO /1gg.>.>.OOF+  kszM^09kZik 	k & s   A; ;	BBr	   r
  r   r   )report_progress_fnmultithreadr'  r   z#File {} was not uploaded correctly!zRetrying...({}/{}))blockr'  r#   )r   r:   r  r
  st_sizer  r   DEFAULT_BUFFER_SIZErc   DXDataObject_get_creation_params_ensure_write_bufsizer  _write_bufsizerR   r   r   r   ra   wait_until_parts_uploadedr   r
   warningro   close)r3   r   r   	keep_openwait_on_closeuse_existing_dxfiler/   r    r(  r%   r   r  r&  retriesmax_retriesr  _remaining_kwargsbufr  r  r"   r   r   r   s   ```    ` `         @@@@@@r   upload_local_filer;    sE   V !tHd';BHHRYY[)11	?o""k k* GKB)M  44 [
 1)G%h/G #//DDVL	%%%9(89)*&GQ'w--.Cc#hF3x1}GMM# .@M_SW&1. -.  	hMthWghJJT"JJ		--/ HHJ}MQ^odh}l|}NA	`  	DKKHUV$3::7KPQ	s   -G G GAH76H7c                     t        d|dd|}t        j                  j                  |      \  }} |j                  | fi | |s |j
                  dd|i| |S )a  
    :param to_upload: String to upload into a file
    :type to_upload: string
    :param media_type: Internet Media Type
    :type media_type: string
    :param keep_open: If False, closes the file after uploading
    :type keep_open: boolean
    :param wait_on_close: If True, waits for the file to close
    :type wait_on_close: boolean
    :returns: Remote file handler
    :rtype: :class:`~dxpy.bindings.dxfile.DXFile`

    Additional optional parameters not listed: all those under
    :func:`dxpy.bindings.DXDataObject.new`.

    Uploads the data in the string *to_upload* into a new file object
    (with media type *media_type* if given) and returns the associated
    remote file handler.

    r  )r   r   r)  r#   )r'   rc   r,  r-  r   r2  )	to_uploadr   r3  r4  r%   r  r8  r9  s           r   upload_stringr>  |  si    0 CJSCFCG ++@@HAGMM)0/0>M>-=>Nr   c                     t        j                  |       j                  ddi      d   }|rfd|D        S fd|D        S )a  
    :param project: Project ID to use as context for the listing
    :type project: string
    :param path: Subtree root path
    :type path: string
    :param recurse: Return a complete subfolders tree
    :type recurse: boolean

    Returns a list of subfolders for the remote *path* (included to the result) of the *project*.

    Example::

        list_subfolders("project-xxxx", folder="/input")

    foldersT)input_paramsc              3   `   K   | ]%  }d k(  s|k(  s|j                  d z         s"| ' yw)/N)
startswith.0fr;   s     r   	<genexpr>z"list_subfolders.<locals>.<genexpr>  s/     eadcka4i1<<X\_bXbKces   #..c              3   n   K   | ],  }|j                        sd |t              dz   d vs)| . yw)rC  r   N)rD  rR   rE  s     r   rH  z"list_subfolders.<locals>.<genexpr>  s9     `aall4.@SPQRUVZR[\]R]R^P_E_`s   555)rc   get_handlerr   )r   r;   recurseproject_folderss    `  r   list_subfoldersrM    sJ      &&w/88yRVFW8XYbcO e?ee`?``r   rC  c                    d }d }|j                         }	|	dk7  r|	j                  d      r|	dd }	|	dk(  rt        dj                  |            t        j
                  j                  |      j                         }
|
dk(  rt        dj                  |            t        t        | |	d	
            }t        |      dk  rt        dj                  |	            |j                          |D ]  } | ||
|	|              t        t        d	d	d	d	d	d	d	            }t        j                  j                  dd| |	d	|      }|y|D ]  }t        j
                  j                   ||
|	|d   d         |d   d         }t        j
                  j!                  |      r|st        dj                  |            t#        j$                  d|d   d   dk(  rdn|d   d   |d   d   |       t'        |d   d   |f|| ||d   d|  y)a  
    :param project: Project ID to use as context for this download.
    :type project: string
    :param destdir: Local destination location
    :type destdir: string
    :param folder: Path to the remote folder to download
    :type folder: string
    :param overwrite: Overwrite existing files
    :type overwrite: boolean

    Downloads the contents of the remote *folder* of the *project* into the local directory specified by *destdir*.

    Example::

        download_folder("project-xxxx", "/home/jsmith/input", folder="/input")

    c                    t         j                  j                  |       set         j                  j                  |       rt	        dj                  |             t        j                  d|        t        j                  |        y y )Nz?Destination location '{}' already exists and is not a directoryz$Creating destination directory: '%s')	r:   r;   isdirexistsr   ro   r
   r   makedirs)ds    r   ensure_local_dirz)download_folder.<locals>.ensure_local_dir  sY    ww}}Qww~~a !"c"j"jkl"mnnLL?CKKN	  r   c                     |dk(  r|dd  n|t        |      dz   d  }t        j                  dk7  r |j                  dt        j                        }|dk7  r t        j                  j                  | |      S | S )NrC  r   r`   )rR   r:   sepreplacer;   rE   )rS  remote_folderremote_subfoldersuffixs       r   compose_local_dirz*download_folder.<locals>.compose_local_dir  sm    )6#)=!!"%CSTWXeTfijTjTkCl66S=^^C0F*0B,rww||Av&=A=r   rC  Nr`   z Invalid remote folder name: '{}'z(Invalid destination directory name: '{}'T)rK  r   zRemote folder '{}' not found)folderr   idr   r   r   r   )r   r   closed)	classnamestater   r]  rK  r   r   r]  r   zHDestination file '{}' already exists but no overwrite option is providedz0Downloading '%s/%s' remote file to '%s' locationr^  )r-   r   r/   r0   )rQ   endswithr   ro   r:   r;   normpathlistrM  rR   sortdictrc   searchfind_data_objectsrE   rQ  r
   r   r6   )r   destdirr]  	overwriter-   r/   r%   rT  r[  normalized_foldernormalized_dest_dirremote_foldersrY  describe_input	files_genremote_filelocal_filenames                    r   download_folderrr    s_   (> C$5$>$>s$C-cr2B<CCFKLL''**7399;b DKKGTUU/'3DdSTN
>a8??@QRSS* f*+>@QScdef T+/)-,0+/,0*."0 1N --hX_5FPT_m . oI 	 ! "&78K8I8CJ8OPX8Y'[ '2*&=f&EG 77>>.))Zaabpq  	G'
3H=Db+V`JabjJk ,V4#	% 	J/5&	""+ '&3(3J(?	" !	""r   )r(   )
Downloaded)	NNNFFNFNT)NFF)T)8__doc__
__future__r   r   r   r   r:   r   r   r  r  r   r   r   collectionsr   rh   r   awscrtr   r   r   rc   r`   r
   r   r   r   rd   r   r   r   r   r   r   compatr   utilsr   rM   concurrent.futures
concurrentr+  r   r'   r6   rH   rZ   ry   r   r   r2   r;  r>  rM  rr  r#   r   r   <module>r|     s  " S R          #         ( D  D  %  "@Z@Z W. F,F,F[_ot > /5.H.HQVfk $")-\0-*^>6  &99%W\!42jX LQSX:>n`"Ha0 .1EVMgMg"'W"r   