5i+>dZddlmZmZmZmZddlZddlZddlZddl Z ddl Z ddl m Z ddl Z ddlZddlmZddlmZddlZddlmZd d lmZmZd d lmZd d lmZd d lmZmZe edZ!dZ"dZ#ejHrdZ#dZ%dZ&dZ'dZ(dZ)ddZ*GddeZ+y)zW DXFile Handler ************** This remote file handler is a Python file-like object. )print_functionunicode_literalsdivisionabsolute_importN)BytesIO)Lock) cpu_count) DXDataObject) DXFileErrorDXIncompleteReadsError)warn)object_exists_in_project) md5_hasher basestringiiii@<i,c|jD]F\}}t|tstd|dt|tr6td|d|d|S)Nz Expected key z of headers to be a stringzExpected value z! of headers (associated with key z) to be a string)items isinstancer ValueError)headerskeyvalues v/home/marpiech/ifpan-abm-pgxpred/analysis/marpiech-gwas-test/venv/lib/python3.12/site-packages/dxpy/bindings/dxfile.py_validate_headersr6s[mmo- U#z*CQR R%, %s,- - - Nc|}td}t|dz}t|dz}t|dz}||krdj||dk7rdSdS||cxkr|krnndj||z S||cxkr|krnndj||z S||cxkr|krnndj||z S||krd j||z Sy ) z'Returns the file size in readable form.iz{0} {1}r bytesbytez {0:.2f} KiBz {0:.2f} MiBz {0:.2f} GiBz {0:.2f} TiBN)floatformat) num_bytesBKBMBGBTBs r_readable_part_sizer*@sA tB rDy B rDy B rDy B2va1f7AA&AA q2##AbD)) q2##AbD)) q2##AbD)) q##AbD)) rc |d}|d}|d}|d}|(||kDr#tdjt||}|4t|t t j t||z }|} || ksJ|r|tjzdk7r'|tj|tjzz z }| tjzdk7r| | tjzz} ||tjzz }n|}|} t| |} t| | } || |z|kr td|r!| tjzdk7r td| S) NmaximumNumPartsminimumPartSizemaximumPartSizemaximumFileSizez"Size of file exceeds maximum of {}rzDpart size would be too small to upload the requested number of bytesz&part size will not be accepted by mmap) r r#r*maxintmathceilr"mmapALLOCATIONGRANULARITYminAssertionError) buffer_size_hintfile_upload_paramsexpected_file_size file_is_mmapd max_num_parts min_part_size max_part_size max_file_sizemin_buffer_sizemax_buffer_size buffer_sizes r_get_write_buf_sizerCTs&'89M&'89M&'89M&'89M%*<}*L>EEFYZgFhijj#O%os499UCU=VYf=f3g/hi#O o -- - T77 71 < t99OdNhNhA[A[.[[+ #Kk?3Kk?3K%;+FI[+[cddt'A'AAQFEFF rceZdZdZdZeejjZ eejjZ eejjZ eejjZeejj Zeejj$Zeejj(Zeejj,Zeejj0Zeejj4Zeejj8Zeejj<Ze Z!ejDjGe!Z$dZ%e&dZ'ddde(e(ddfdZ)d"d Z*d Z+d Z,d Z-d Z.e/Z/e/Z0d"dZ1e2jffdZ4dZ5d#dZ6dZ7dZ8d#dZ9d#dZ:dZ;d$dZdZ?d'dZ@ d(dZAd$dZBd)dZCd)dZDd$d ZEd$d!ZFy)*DXFileaeRemote file object handler. :param dxid: Object ID :type dxid: string :param project: Project ID :type project: string :param mode: One of "r", "w", or "a" for read, write, and append modes, respectively. Use "b" for binary mode. For example, "rb" means open a file for reading in binary mode. :type mode: string .. note:: The attribute values below are current as of the last time :meth:`~dxpy.bindings.DXDataObject.describe` was run. (Access to any of the below attributes causes :meth:`~dxpy.bindings.DXDataObject.describe` to be called if it has never been called before.) .. py:attribute:: media String containing the Internet Media Type (also known as MIME type or Content-type) of the file. .. automethod:: _new file) max_workersNO_PROJECT_HINTctdy)z+ .. deprecated:: 0.191.0 z&set_http_threadpool_size is deprecatedN)print)cls num_threadss rset_http_threadpool_sizezDXFile.set_http_threadpool_sizes  67rNFctj|||d|_|d|_n6d|vrd|_|j dd}|dvr t d|d k(|_t |_t |_||_ d|_ ||_ ||_ ||_ d \|_|_|_t#|_d \|_|_t+|_d |_d|_d |_d |_d|_y)a :param dxid: Object ID :type dxid: string :param project: Project ID :type project: string :param mode: One of "r", "w", or "a" for read, write, and append modes, respectively. Add "b" for binary mode. :type mode: string :param read_buffer_size: size of read buffer in bytes :type read_buffer_size: int :param write_buffer_size: hint for size of write buffer in bytes. A lower or higher value may be used depending on region-specific parameters and on the expected file size. :type write_buffer_size: int :param expected_file_size: size of data that will be written, if known :type expected_file_size: int :param file_is_mmapd: True if input file is mmap'd (if so, the write buffer size will be constrained to be a multiple of the allocation granularity) :type file_is_mmapd: bool )dxidprojectFNTb)rwaz[mode must be one of 'r', 'w', or 'a'. Character 'b' may be used in combination (e.g. 'wb').rTNNNNNrr )r __init__ _binary_mode_close_on_exitreplacerr _read_buf _write_buf _read_bufsize_write_bufsize_write_buffer_size_hint_expected_file_size_file_is_mmapd _download_url_download_url_headers_download_url_expiresr_url_download_mutex_request_iterator_response_iteratorset_http_threadpool_futures_pos _file_length _cur_part_num_uploaded_parts_exists_in_proj)selfrOrPmoderead_buffer_sizewrite_buffer_sizer:r;s rrXzDXFile.__init__s0 dw?" <"&D d{$(!||C,?* !~#'3;D  !)- #'8$#5 +VfRD68R$(6 :D7 7(+%  #$ #rc |||d<tjj|fi|}|j|d|dy)aV :param dx_hash: Standard hash populated in :func:`dxpy.bindings.DXDataObject.new()` containing attributes common to all data object classes. :type dx_hash: dict :param media_type: Internet Media Type :type media_type: string Creates a new remote file with media type *media_type*, if given. NmediaidrP)dxpyapifile_newset_ids)rpdx_hash media_typekwargsresps r_newz DXFile._newsD  !)GG xx  3F3 T$Z!34rc|SNrps r __enter__zDXFile.__enter__s rc|j|jr%|jdk(r|jyyy)Nopen)flushrZ _get_stateclose)rptyper tracebacks r__exit__zDXFile.__exit__s3   4??#4#> JJL$? rct|dsy|jjdkDst|jdkDr7t dt dt dt dt d |j d y#t$rB}t d |jtjtjd}~wwxYw) a# Exceptions raised here in the destructor are IGNORED by Python! We will try and flush data here just as a safety measure, but you should not rely on this to flush your data! We will be really grumpy and complain if we detect unflushed data here. Use a context manager or flush the object explicitly to avoid this. In addition, when this is triggered by interpreter shutdown, the thread pool is not available, and we will wait for the request queue forever. In this case, we must revert to synchronous, in-thread flushing. We don't know how to detect this condition, so we'll use that for all destructor events. Neither this nor context managers are compatible with kwargs pass-through (so e.g. no custom auth). r]Nrz=== WARNING! ===zCThere is still unflushed data in the destructor of a DXFile object!zcWe will attempt to flush it now, but if an error were to occur, we could not report it back to you.z@Your program could fail to flush the data but appear to succeed.zxInstead, please call flush() or close(), or use the context managed version (e.g., with open_dxfile(ID, mode='w') as f:)F) multithreadz@=== Exception occurred while flushing accumulated file data for ) hasattrr]telllenrjrr Exception_dxidrprint_exceptionsysexc_info)rpes r__del__zDXFile.__del__s t\*  ??   !A %T-J-J)Ka)O # $ V W v w S T L M  JJ5J )  Y]YcYce f  % %s||~ 6  s;B C=CCc#Kd}|jr td |j|j}|sn?||z }|j d}t t |dz D] }|| |d}]|r|yyw)NrRz1Cannot read lines when file opened in binary mode r )rYr readr^splitranger)rp_buffermorepartsis r__iter__zDXFile.__iter__?s   QR R99T//0D tOGMM$'E3u:>* Ah BiG M sBBc|j|jtj|||d|_d|_d|_d|_y)aK :param dxid: Object ID :type dxid: string :param project: Project ID :type project: string Discards the currently stored ID and associates the handler with *dxid*. As a side effect, it also flushes the buffer for the previous file object if the buffer is nonempty. Nrr )rrr rzrkrlrmrn)rprOrPs rrzzDXFile.set_idsSsI :: ! JJLT41  #$ rc|tjk(rd}n}|tjk(r |j}n]|tjk(r?|j dk(r#|j }t|d|_|j }n td|j}||z|_d}|jj}||kr |||z kDr9d}n6tjj|j}||z ||z kDrd}|r"|jj||z |zy||k(ryt|_ d\|_|_y)at :param offset: Position in the file to seek to :type offset: integer Seeks to *offset* bytes from the beginning of the file. This is a no-op if the file is open for writing. The position is computed from adding *offset* to a reference point; the reference point is selected by the *from_what* argument. A *from_what* value of 0 measures from the beginning of the file, 1 uses the current file position, and 2 uses the end of the file as the reference point. *from_what* can be omitted and defaults to 0, using the beginning of the file as the reference point. rNsizez$Invalid value supplied for from_whatFTrW)osSEEK_SETSEEK_CURrkSEEK_ENDrldescriber1r r\rrwutilsstring_buffer_lengthseekrrgrh) rpoffset from_what reference_posdescorig_posin_buf orig_buf_posbuf_lens rrz DXFile.seekis5  #M "++ % IIM "++ %  D(}}$'V $5! --MDE E99!F* ~~**, H h//jj55dnnEG%(99  NN   x 7& @ A x  %YDN>H ;D "D$;rc|jS)z Returns the current position of the file read cursor. Warning: Because of buffering semantics, this value will **not** be accurate when using the line iterator form (`for line in file`). )rkrs rrz DXFile.tellsyyrc p|jjdkDr~|jj}t|_|r |j|fd|j i|n|j ||j fi||xj dz c_t|jdkDrotjj|j |jD]%}|jdk7s|j t|_yy#t|_wxYw)z4 Flushes the internal write buffer. rindexr N)r]rgetvaluer_async_upload_part_requestrm upload_partrrjrwrwait_for_all_futures exceptionri)rprr}datafutures rrz DXFile.flushs ??   !A %??++-D%iDO///UDNNUfU   t~~@@ NNa N t,, - 1 JJ + +D,I,I J 6";;1F'')T1$..00114- 214-s"D$D$$D5ct|j|jk\rtjj |j}|j dk7r|j |jj|t|j|jk\r|jj|jg|i|}|jj|yr) rrj_http_threadpool_sizerwrwait_for_a_futurerremove_http_threadpoolsubmitradd)rpargsr}rs rrz!DXFile._async_upload_part_requests$//0D4N4NNZZ11$2O2OPF!T)&&((  ) ) 0 0 8 $//0D4N4NN .&&--d.>.>PPP %%))&1rc |jytjj|j dddiifi|d}|d|_t |j||j|j|_y)NfieldsfileUploadParametersTemptyLastPartAllowed) r_rwrxproject_describe get_proj_id_empty_last_part_allowedrCr`rarb)rpr}r9s r_ensure_write_bufsizezDXFile._ensure_write_bufsizes    * !XX66     .5 6   ! " );;Q(R%1$2N2N2D262J2J262E2EGrc Ht|tsJjdifd}jj dk(r!j t |k(r ||yj jj z }t ||krjj|yjj|d|jj}t_||j||dfiy) :param data: Data to be written :type data: str or mmap object :param multithread: If True, sends multiple write requests asynchronously :type multithread: boolean Writes the data *data* to the file. .. note:: Writing to remote files is append-only. Using :meth:`seek` does not affect where the next :meth:`write` will occur. cr j|fdjinj|jfixjdz c_y)Nrr )rrmr)data_for_write_reqr}rrps r write_requestz%DXFile._write2..write_requestsP///0Bc$..c\bc   !3T^^NvN NNa NrrNr) rr rr]rr_rwriterr)rprrr}rremaining_space temp_datas` `` r_write2zDXFile._write2s$&'&""",V,  ??   !Q &4+>+>#d)+K $  --0D0D0FF t9 ' OO ! !$ ' OO ! !$'7"8 9002I%iDO ) $ DJJtO,- 8 8rc t|tr|jd}nxt|tr t |}n\t|tr|}nIt|t j r t |}n#t djt|t|tsJ|j|fd|i|y)rutf-8z'Invalid type {} for write data argumentrN) rstrencode bytearrayr r4r r#rr)rprrr}bts rrz DXFile.writes2 dC W%B i (tB e $B dii (tBGNNtTXzZ[ ["e$%$ R;[;F;rc 8|jdddhi|ddk(S)a :returns: Whether the remote file is closed :rtype: boolean Returns :const:`True` if the remote file is closed and :const:`False` otherwise. Note that if it is not closed, it can be in either the "open" or "closing" states. rstateclosedr)rrpr}s rrz DXFile.closed<s)t}}8WI88AXMMrc z|jdi||jdi||jdk(r|jr |jdi|d|vr|d=t jj|jfi||r|jdi|yy#t j j$rYfwxYw)aJ :param block: If True, this function blocks until the remote file has closed. :type block: boolean Attempts to close the file. .. note:: The remote file cannot be closed until all parts have been fully uploaded. An exception will be thrown if this is not the case. rreport_progress_fnNr)rr ) rrrnrrrw exceptions InvalidStaterx file_closer_wait_on_close)rpblockr}s rrz DXFile.closeHs  V #"",V,  # #q (T-J-J     262 6 )+, DJJ1&1  D   )& ) ??//  sBB:9B:c *|j|fi|y)a+ :param timeout: Maximum amount of time to wait (in seconds) until the file is closed. :type timeout: integer :raises: :exc:`dxpy.exceptions.DXFileError` if the timeout is reached before the remote file has been closed Waits until the remote file is closed. N)r)rptimeoutr}s r wait_on_closezDXFile.wait_on_closejs G.v.rc T t|tsJi |t| d<t}t |drZt |drN|j } |j t}|r|j|nn+|j|n|j||j d<t| d< fd} tj| |d d dtdd xjd z c_|r t!d ||t|yy)a :param data: Data to be uploaded in this part :type data: str or mmap object, bytes on python3 :param index: Index of part to be uploaded; must be in [1, 10000] :type index: integer :param display_progress: Whether to print "." to stderr when done :type display_progress: boolean :param report_progress_fn: Optional: a function to call that takes in two arguments (self, # bytes transmitted) :type report_progress_fn: function or None :raises: :exc:`dxpy.exceptions.DXFileError` if *index* is given and is not in the correct range, :exc:`urllib3.exceptions.HTTPError` if upload fails Uploads the data in *data* as part number *index* for the associated file. If no value for *index* is given, *index* defaults to 1. This probably only makes sense if this is the only part to be uploaded. NrrrTmd5rcdvrtjd<nddkDr dxxdzcc<dvr td<tjjj fi}|d}|t |jdifS)N max_retriesrr rurlr)rwDEFAULT_RETRIESFILE_REQUEST_TIMEOUTrx file_uploadrrget)r~rr} req_inputrps rget_upload_url_and_headersz6DXFile.upload_part..get_upload_url_and_headerssF*(,(<(<}% &*}%*%&$8y!88'' IHHDu+C)$((9b*ABB BrFPUT) jsonify_data prepend_srv always_retryrauthmethodr .)rr r1rrrrMD5_READ_CHUNK_SIZEupdater hexdigestrrw DXHTTPRequest#PART_UPLOAD_REQUEST_TIMEOUT_SECONDSrnr) rprrdisplay_progressrr}rrewind_input_buffer_offset bytes_readrrs ` ` @rrzDXFile.upload_partts"$&'&  !$UIg l 4 WT6%:)- &!YY':; JJz*  II0 1 JJt ==? %I & C& 5(-',(,#F $"' )   A%  I  ) tSY / *rc (|jdi|y)Nr)_wait_until_parts_uploadedrs rwait_until_parts_uploadedz DXFile.wait_until_parts_uploadeds'''1&1rc |j5|j"|jtjkrGd|i}|||d<|||d<|@dtj vr.|j }|rt|j|r|}||tjur||d<dtj vrLttj dd5}|$|tjk7r|j|dddd |vr t|d <tjj |j"|fi|} | d |_t%| j'd i|_|r| d d z dz |_nd|_|j} t+j*|j(} ddd| | fS#1swYxYw#1swY  fSxYw)a :param duration: number of seconds for which the generated URL will be valid, should only be specified when preauthenticated is True :type duration: int :param preauthenticated: if True, generates a 'preauthenticated' download URL, which embeds authentication info in the URL and does not require additional headers :type preauthenticated: bool :param filename: desired filename of the downloaded file :type filename: str :param project: ID of a project containing the file (the download URL will be associated with this project, and this may affect which billing account is billed for this download). If no project is specified, an attempt will be made to verify if the file is in the project from the DXFile handler (as specified by the user or the current project stored in dxpy.WORKSPACE_ID). Otherwise, no hint is supplied. This fall back behavior does not happen inside a job environment. A non preauthenticated URL is only valid as long as the user has access to that project and the project contains that file. :type project: str :returns: download URL and dict containing HTTP headers to be supplied with the request :rtype: tuple (str, dict) :raises: :exc:`~dxpy.exceptions.ResourceNotFound` if a project context was given and the file was not found in that project context. :raises: :exc:`~dxpy.exceptions.ResourceNotFound` if no project context was given and the file was not found in any projects. Obtains a URL that can be used to directly download the associated file. Npreauthenticateddurationfilename DX_JOB_IDrP_DX_DUMP_BILLED_PROJECTrTrrrexpiresirlL")rfrcretimerenvironrrget_idrErHrrrrwrx file_downloadrrrrdcopy) rpr r rrPr}rproject_from_handlerfdr~retval_download_urlretval_download_url_headerss rget_download_urlzDXFile.get_download_urlsB % %8 P!!)T-G-G$))+-U*,<=''/D$''/D$?{"**'D+/+;+;+=(+0HXl0m"6&7&:P:P+P&-DO- :bjj)BCSI.R".7f>T>T3THHW-.F*(txx SU?V-W*#15i1E1JD.1.chunk_ranges2s'O 2BCNA!G+ #On$Dq$H' R %}44>1!$44=V9V[twx[x9y%($)>@P%QNQ "G+s AA" A"rPr r)ir ) r^rlrr1r rrw_dxhttp_read_ranger6r) rprrrPr&r}rr'chunk_start_pos chunk_end_posrrs ` r_generate_read_requestszDXFile._generate_read_requests"s  ##11     $ 4==*6*D #DL 1D  d?''G T&& &/0 0 /;9g.N F *O]0400KKFKLC))C/3}^b^o^ors^sKt,@,BCEF F FsCCc|j;tjj|j|j ||_ t |jS#d|_d|_xYw)N)do_first_task_sequentially)rhrwrresponse_iteratorrgrnext)rpget_first_chunk_sequentiallys r_next_response_contentzDXFile._next_response_contentDsr  " " *&*jj&B&B&&%%+G'C'D #  //0 0 '+D #%)D " s AA/c |jdk(r8|jd i|}|ddk7r tdt|d|_|jdkDxr!|jdk(xrt j }|j|jk(ry|dk(s||j|jz kDr|j|jz }|j}t jj||jz }||kr&|xj|z c_|j|S|j} |j} |jdtj|xj|z c_|j| |zkr| |z|jz } |j$|j d |j|d ||_|j%| } t'| | kr0|j)| |xjt'| z c_nq|j)| d| |xj| z c_t+|_|jj)| | d|jjd|j| |zkr|j| |jS) a :param length: Maximum number of bytes to be read :type length: integer :param project: project to use as context for this download (may affect which billing account is billed for this download). If specified, must be a project in which this file exists. If not specified, the project ID specified in the handler is used for the download, IF it contains this file. If set to DXFile.NO_PROJECT_HINT, no project ID is supplied for the download, even if the handler specifies a project ID. :type project: str or None :rtype: string :raises: :exc:`~dxpy.exceptions.ResourceNotFound` if *project* is supplied and it does not contain this file Returns the next *length* bytes, or all the bytes until the end of file (if no *length* is given or there are fewer than *length* bytes left in the file). .. note:: After the first call to read(), the project arg and passthrough kwargs are not respected while using the same response iterator (i.e. until next seek). Nrrz5Cannot read from file until it is in the closed staterirr)rrP)r1r)rlrr r1rkrwJOB_IDr\rrrrrrrrhr,rgr2rrr) rplengthuse_compressionrPr}rr1bufbuf_remaining_bytesr orig_file_pos remaining_lencontents r_read2z DXFile._read2Use2    $ 4==*6*DG}(!"YZZ #DL 1D )-(9(9J(F(i499XY>(i^b^i^i$ 99)) ) T>Vd&7&7$))&CC&&2Fnn"jj==cBSXXZO ( ( II I88F# #88:L IIM HHQ $ II, ,I))mf44 - 6 B **2-IT-I-I.H"&))W.H@F.HD*55So5pw<-/IIg&IIW-IIIgn}56II.I%,YDNNN(()@ANN''*#))mf44$ HH\ "88: rc B|I|j=|j-t|j|j|_||jr|j}|jd|||d|}|j dur|S|j dS)N)r5r6rPTrr)rorrrr<rYdecode)rpr5r6rPr}rs rrz DXFile.reads ?t33;@P@P@R@^#;DKKM4K[K[K]#^D ?t33&&(Gt{{e&/SZe^de    $K{{7##rctjj|j|j g|dy)a. :param all_copies: Force the transition of files into the archived state. Requesting user must be the ADMIN of the project billTo org. If true, archive all the copies of files in projects with the same billTo org. :type all_copies: boolean :raises: :exc:`~dxpy.exceptions.InvalidState` if the file is not in a live state :raises: :exc:`~dxpy.exceptions.PermissionDenied` if the requesting user does not have CONTRIBUTE access or is not an ADMIN of the project billTo org with allCopies=True. )files allCopiesN)rwrxproject_archiverr)rp all_copiess rarchivezDXFile.archives/   !1!1!3 ]g5hirctjj|j|j g|dy)ao :param dry_run: If true, only display the output of the API call without executing the unarchival :type dry_run: boolean :raises: :exc:`~dxpy.exceptions.InvalidState` if the file is not in a closed or archived state :raises: :exc:`~dxpy.exceptions.PermissionDenied` if the requesting user does not have CONTRIBUTE access )r@dryRunN)rwrxproject_unarchiverr)rpdry_runs r unarchivezDXFile.unarchives/ ""4#3#3#5$++-\c7derr)TF)i: )NFN)NFNN)rNNNrV)G__name__ __module__ __qualname____doc___class staticmethodrwrx file_describe _describefile_add_types _add_typesfile_remove_types _remove_typesfile_get_details _get_detailsfile_set_details _set_detailsfile_set_visibility_set_visibility file_rename_renamefile_set_properties_set_properties file_add_tags _add_tagsfile_remove_tags _remove_tagsr_closefile_list_projects_list_projectsDXFILE_HTTP_THREADSrrget_futures_threadpoolrrH classmethodrMDEFAULT_BUFFER_SIZErXrrrrrr0__next__rzrrrrrrrrrrrrrr rr,r2r<rrDrIrrrrErEs4FTXX334Idhh556J !;!;rts"SR%%% % <5+)+q)";;'!&)#*(.b} f\} fr