
    5i0                     P   d Z ddlmZmZmZmZ ddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlmZ ddlZddlZddlmZ ddlmZmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddl m!Z! ddl"Z"ejF                  ddddfdZ$d Z%d Z&d Z'ejF                  ddddfdZ(d Z)y)z
Helper Functions
****************

The following helper functions are useful shortcuts for interacting with Database objects.

    )print_functionunicode_literalsdivisionabsolute_importN)defaultdict   )logger   )dxfileDXFile)
dxdatabase
DXDatabase)FILE_REQUEST_TIMEOUT)
md5_hasher)DXFileErrorDXChecksumMismatchErrorDXIncompleteReadsErrorerr_exit)response_iteratorFc	           	      X    t        d       }
d}|st        | ||||
f|||||d|	}|syy)av  
    :param dxid: DNAnexus database ID or DXDatabase (database handler) object
    :type dxid: string or DXDatabase
    :param dst_filename: Local filename
    :type dst_filename: string
    :param src_filename: Name of database file or folder being downloaded
    :type src_filename: string
    :param file_status: Metadata for the source file being downloaded
    :type file_status: dict
    :param append: If True, appends to the local file (default is to truncate local file if it exists)
    :type append: boolean
    :param project: project to use as context for this download (may affect
            which billing account is billed for this download). If None or
            DXFile.NO_PROJECT_HINT, no project hint is supplied to the API server.
    :type project: str or None
    :param describe_output: (experimental) output of the database-xxxx/describe API call,
            if available. It will make it possible to skip another describe API call.
            It should contain the default fields of the describe API call output and
            the "parts" field, not included in the output by default.
    :type describe_output: dict or None

    Downloads the remote file referenced by *src_filename* from database referenced
    by *dxid* and saves it to *dst_filename*.

    Example::

        download_dxdatabasefile("database-xxxx", "localfilename", "tablename/data.parquet)

    c                       y)N    r       /home/marpiech/ifpan-abm-pgxpred/analysis/marpiech-gwas-test/venv/lib/python3.12/site-packages/dxpy/bindings/dxdatabase_functions.py<lambda>z)download_dxdatabasefile.<locals>.<lambda>M   s    r   F)	chunksizeappendshow_progressprojectdescribe_outputN)r   _download_dxdatabasefile)dxiddst_filenamesrc_filenamefile_statusr   r   r   r    r!   kwargspart_retry_countersuccesss               r   download_dxdatabasefiler*   ,   sU    B %Y/G*4+7+7+6+=	
5
 6?289F3:;J
5 .4
5 r   c                     d }t         j                  d   j                  t         j                        D ].  }t         j                  j                  ||       } ||      s,|c S  y )Nc                     t         j                  j                  |       xr$ t        j                  | t         j                        S N)ospathisfileaccessX_OK)fpaths    r   is_exez_which.<locals>.is_exe`   s)    ww~~e$B5"'')BBr   PATH)r.   environsplitpathsepr/   join)programr4   r/   exe_files       r   _whichr<   _   sW    C 

6"((4 77<<g.(O r   c                    t        d      }|t        d       |d| g}	 t        d       t        j                  |      }j                         j                         }t        |      dk7  rt        dt        |      z          |d   }|j                  d	      }||k7  r&t        d
t        |      z   dz   t        |      z          t        d       y # t        j
                  $ r t        dt        |      z          Y w xY w)Nmd5sumz&md5sum is not installed on this systemz-bzCalculating checksumzFailed to run md5sum: r   zmd5sum returned weird results: r   asciizChecksum doesn't match z  expected:zChecksum correct)r<   r   print
subprocesscheck_outputCalledProcessErrorstrstripr7   lenencode)filename	md5digest
md5sum_execmdcmd_outline
actual_md5s          r   _verifyrO   k   s    !J9:tX
&C6$%))#. ==?  "D
4yA~2SY>?aJ  )I Y*S_<}LsS\~]^	
 (( 6)CH456s    C *C<;C<c                 .    t        j                  |        y r-   )r	   debug)msgs    r   do_debugrS      s    
LLr   c
           	      L  
  d#fd	}d}t        | t              r| n"t        | t        j                  k7  rnd      t	        dj                               |	r|	j                  d      |	}n j                  d$dhdd
}d	d
|d   dit        t               |d   }d} D ]  }||   d<   ||   d   z  } t	        dj                               t               t        j                  j                        }|j                  d      }|dk7  r|d| }t        |       t	        dj                  |             |rt        |d      }n	 t        |d      }|r	 |dd       
fd fd}fd}|5  	 |dkD  xr t"        j$                  }d\  }}}t'         |       j(                  |      D ]b  \  }}||k7  r|dt+               }}}|t-        |      z  }|j/                  |       |j1                  |       |sL|t-        |      z  } |||       d |r |||d       |rt:        j<                  j1                  d"       	 ddd       y# t         $ r t        |d      }Y w xY w# t2        $ r t5        t7        j8                         t:        j<                         |xx   dz  cc<   ||   dkD  rHt5        d j                   j>                         ||   |      t:        j<                         Y ddd       y! w xY w# 1 sw Y   yxY w)%au  
    Core of download logic. Download the specified file/folder *src_filename*
    associated with database-id *dxid* and store it in
    a local file *filename*.

    The return value is as follows:
    - True means the download was successfully completed
    - False means the download was stopped because of a retryable error
    - Exception raised for other errors
    c                 z   d}|xs d}| |kD  r| }t        t        | t        |      z  |z              }t        t        j                  | t        |      z  dz              }d}t
        j                  j                  d       t
        j                  j                  |j                  ||dkD  rd|dz
  z  dz   nd	d
||z
  z  | |rdj                  |      nd	|             t
        j                  j                          t
        j                  j                  d       t
        j                  j                          y )N<   r
   d   zN[{done}{pending}] {action} {done_bytes:,}{remaining} bytes ({percent}%) {name}z[2Kr   =>  z of {size:,})size)actiondonepending
done_bytes	remainingpercentname)
introundfloatmathfloorsysstderrwriteformatflush)	bytes_downloaded	file_sizer]   	num_tickseffective_file_sizeticksrb   fmtrH   s	           r   print_progressz0_download_dxdatabasefile.<locals>.print_progress   s   	'n111"2E+e4G.HHIUVWdjj"2U;N5O"OSV!VWX^ 	

"

6FKai#*;c*AUW,/9u3D,E/?W`n.C.C.C.Sfh,3)1 $ 3 	4 	





r   r   N)r    zAdxdatabase_functions.py _download_dxdatabasefile - dxdatabase: {}partsT)fieldsdefault_fields1completer\   )stater\   )keystartz;dxdatabase_functions.py _download_dxdatabasefile - parts {}/z?dxdatabase_functions.py _download_dxdatabasefile - dest_path {}abzrb+wbc                     t        dj                  |||               j                  d
	d\  }}d}t        j                  ||||t
        |      }| |fS )Nz@dxdatabase_functions.py get_chunk - start {}, end {}, part id {})r%   r    Fr   )rS   rm   get_download_urldxpy_dxhttp_read_ranger   )part_id_to_getr}   endurlheaders	sub_rangedatar   r'   r    r%   s          r   	get_chunkz+_download_dxdatabasefile.<locals>.get_chunk   sl    SZZ[`beguvw2z22hV]haghW	&&sGUCAUW`at##r   c               3      K   D ]J  } |    }t        |d   |d   |d   z         D ]'  }t        |z   |d   |d   z         dz
  }| ||gi f ) L y w)Nr}   r\   r
   )rangemin)part_id_to_chunk	part_infochunk_start	chunk_endr   r   rv   parts_to_gets       r   chunk_requestsz0_download_dxdatabasefile.<locals>.chunk_requests   s      , 	P./I$Yw%779KiX^N_9_ajk Pi 779KiX^N_9_`cdd	"2K!KROOP	Ps   AAc                    |A||    d   k7  r6d}|j                   j                         | |    d   |      }t        |      |;d|    vr4t        j                  dj                   j                                      y |^|j                         |    d   k7  rDd}|j                   j                         | |    d   |j                               }t        |      y y )Nr\   z=Unexpected part data size in {} part {} (expected {}, got {})md5z6Download of file {} is not being checked for integrityz5Checksum mismatch in {} part {} (expected {}, got {}))rm   get_idDXPartLengthMismatchErrorwarningswarn	hexdigestr   )_part_id	got_byteshasherrR   r   rv   s       r   verify_partz-_download_dxdatabasefile.<locals>.verify_part   s     Y%/&2I%IQC**.Z..0(E(OF<SU^_C+C00%uX">MMRYYZkZdZkZkZmnoF$4$4$6%/%:P$PIC**.Z..0(E(OE<RTZTdTdTfgC)#.. %Qr   i   )NNN)do_first_task_sequentially	Completed)r]   )filer
   z)Retrying {} ({} tries remain for part {})F
)
Downloadedr   ) 
isinstancer   r   NO_PROJECT_HINTrS   rm   getdescribesortedre   ensure_local_dirr.   r/   r9   rfindopenIOErrorr   JOB_IDr   _http_threadpoolr   rF   updaterl   r   r@   	traceback
format_excrj   rk   r   )!r#   rH   r%   r&   r(   r   r   r   r    r!   r'   ru   _bytesdxdatabase_descrp   offsetpart_id	dest_pathdest_dir_idxdest_dirfhr   r   get_first_chunk_sequentiallycur_partr   r   
chunk_part
chunk_datar   r   rv   r   s!    ``  `  ` `                  @@@@r   r"   r"      sC   2 F$
#
'VE[E[:[waeg
PWWXbcd?..w7C)-*--^gYt^W]^ kK4GHIE%S)LF#IF )"(gw%.(() JQQRWXY XX|4I??3'Lr]l+"NUUV_`a)T"	'i'B q$$P
/ 
 $	,5
,B,Rt{{(*:'Hi*;N<L<F<W<WWs+u 6&
J ) 3=aiHS_,	j)$ c*o-F"6956  vyE JJT"I$ $C  	'i&B	'r  	)&&(szz:x(A-(!(+a/AHHIZIZIZI\^pqy^z  }E  F::'=$ $> 	1$ $sJ   I& LBJ&J;"L&I?>I?B	LLLLL#c                    t         j                  j                  |       set         j                  j                  |       rt	        dj                  |             t        j                  d|        t        j                  |        y y )Nz?Destination location '{}' already exists and is not a directoryz$Creating destination directory: '%s')	r.   r/   isdirexistsr   rm   r	   rQ   makedirs)ds    r   r   r     sY    77==77>>!_ffghijj;Q?
A	 r   )*__doc__
__future__r   r   r   r   r.   rj   rh   mmapstathashlibr   r   collectionsr   multiprocessingr   rZ   r	   r   r   r   r   r   compatr   
exceptionsr   r   r   r   utilsr   rA   DEFAULT_BUFFER_SIZEr*   r<   rO   rS   r"   r   r   r   r   <module>r      s   " S R          #     $ (  _ _ % U[UoUox}  NS$($.5f,  &99%W\!4Unr   