5iIddlmZmZmZmZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl mZmZmZmZmZmZddlmZmZmZmZmZddlmZddlmZdd lmZd Z d Z!d Z"d Z#idfdZ$ddZ%dZ&dZ'dZ(dZ)gdfdZ*ddZ+dZ,y))print_functionunicode_literalsdivisionabsolute_importN)get_field_from_jborget_job_from_jborget_index_from_jbor is_job_refjob_output_to_str JOB_STATES)GREENBLUEBOLDENDCfill)is_localjob_id) basestring)file_load_utilsw c|dz }|dttjjdzz }t j |y)z :param msg: string message to print before exiting Print the error message, as well as a blurb on where to find the job workspaces rz&Local job workspaces can be found in: DX_TEST_JOB_HOMEDIRSN)strosenvirongetsysexit)msgs }/home/marpiech/ifpan-abm-pgxpred/analysis/marpiech-gwas-test/venv/lib/python3.12/site-packages/dxpy/utils/local_exec_utils.pyexit_with_errorr#!s;4KC 3c"**..I_:`6a aaCHHSMcLg}|D]k}t||r t||jds/yt||tst||t sX|j ||mt|dkDr#|j}t|trtt|D]k}t||r t||jds/yt||tst||t sX|j ||mnp|D]k}t||r t||jds/yt||tst||t sX|j ||mt|dkDr#y)z :param io_hash: input/output hash :type io_hash: dict :returns: boolean indicating whether any job-based object references are found in *io_hash* localjobTrF) r r startswith isinstancelistdictappendlenpoprange)io_hashqfieldthingis r"has_local_job_refsr4,si A% gen % 0;;JG  -GEND1Q HHWU^ $ % a&1* eT "3u:& 'eAh'(q2==jI#a$/:eAh3MHHU1X&  ' +eEl+(u6AA*M#e d3z%,PT7UHHU5\*  + a&1*" r$Tc" t|}t|t|  fd}t|rO|j ||rt d|zdz|S||vrt d|zdzzdz|||St j|} |j|j}|d vrt d|zdzzdz||d S#t $r"}t d|zd zt|zd}~wwxYw) a :param jbor: a dict that is a valid job-based object reference :type jbor: dict :param job_outputs: a dict of finished local jobs to their output hashes :type job_outputs: :class:`collections.OrderedDict` :returns: the referenced value if present :raises: :exc:`Exception` if the job-based object reference cannot be resolved TODO: Support metadata references c"|S|SN) output_hash ref_job_field ref_job_indexs r"resolve_from_hashz*resolve_job_ref..resolve_from_hash\s$  }- -}-m< >#  2 2@:MPbberrvUUV V (!344  Z1J>ORUVWRXXY Y ZsC## D,D  Dcg}|D]_}t||rt||||||<%t||tst||tsL|j ||at |dkDr |j}t|trwtt |D]_}t||rt||||||<%t||tst||tsL|j ||and|D]_}t||rt||||||<%t||tst||tsL|j ||at |dkDr yy)a :param io_hash: an input or output hash in which to resolve any job-based object references possible :type io_hash: dict :param job_outputs: a mapping of finished local jobs to their output hashes :type job_outputs: dict :param should_resolve: whether it is an error if a job-based object reference in *io_hash* cannot be resolved yet :type should_resolve: boolean Modifies *io_hash* in-place. rN) r rLr(r)r*r+r,r-r.)r/rFrGr0r1r2r3s r"resolve_job_referencesrNusX A% gen %,WU^[.YGEN  -GEND1Q HHWU^ $ % a&1* eT "3u:& 'eAh'.uQxnUE!Ha$/:eAh3MHHU1X&  '  +eEl+#25<n#]E%Le d3z%,PT7UHHU5\*  + a&1*r$c d}t|trDd|vr@t|dtr|d}n't|dtr|djd}|yt j |}|jddk7r|Sy)Nz$dnanexus_linkidstateclosed)r(r*rrrArD)r2obj_idobj_descs r"get_nonclosed_data_obj_linkrUs F%#3u#< e,-z :+,F ./ 6+,006F ~}}V$H||G( )r$cg}|D]_}t||}||j|%t||tst||tsL|j||at |dkDr |j }t|trwtt |D]_}t||}||j|%t||tst||tsL|j||and|D]_}t||}||j|%t||tst||tsL|j||at |dkDr yy)zS Add DNAnexus links to non-closed data objects in input_hash to depends_on Nr)rUr+r(r)r*r,r-r.) input_hash depends_onr0r1 possible_depr2r3s r"get_implicit_depends_onrZs] A(2:e3DE  #   l +  5)4 0Jz%?PRV4W HHZ& ' ( a&1* eT "3u:& ':58D +%%l3a$/:eAh3MHHU1X&  ' +:5<H +%%l3e d3z%,PT7UHHU5\*  + a&1*r$ct|dkDrttd|D]} td|zdz|jdr"|j |}t d|zdz|jdr&t j|}|jn5t j|}|j}|jyy#t $r"}t d |zd zt|zd}~wwxYw) NrzKProcessing dependsOn and any DNAnexus links to closing objects in the inputz Waiting for z...r&r=z* could not be found in local finished jobsjobr>: ) r,printrr'rr@rArBrC get_handlerrD_wait_on_closer)rXall_job_outputsan_idrIhandlerdescrJs r"wait_for_depends_onres : d`ab OE O&.67##J/&**519'9e(eff%%e, JJu-E&&("..u5G"++-D**, O O 5 = Ds1v MNN OsB0C D&DDcndD]0}|tjvstjd|zdz2y)N) DX_FS_ROOTDX_TEST_CODE_PATHrzEError: Cannot run an entry point locally if the environment variable z has not been set)rrrr )vars r"ensure_env_varsrjs<(z bjj HH\_bbexx y zr$c&ttjjtjdd}t |d5}t j|tj}dtt|z}dddt |t5}d<t j||d|jtdddtjjtjd}tj |tjjtjdd } t | d5}t j|} ddd||||d } ||| d < j#| t | t5}t j| |d|jtddd|S#1swY]xYw#1swYxYw#1swYxYw#1swY|SxYw) a :param function: function to run :param input_hash: input to new job :param depends_on: list of data object IDs and/or job IDs (local or remote) to wait for before the job can be run :type depends_on: list of strings :param name: job name (optional) :returns: new local job ID This function should only be called by a locally running job, so all relevant DX_TEST_* environment variables should be set. This function will set up the home directory for the job, add an entry in job_outputs.json, and append the job information to the job_queue.json file. (Both files found in $DX_TEST_JOB_HOMEDIRS.) rjob_outputs.jsonrobject_pairs_hookz localjob-Nindentjob_queue.json)rPfunctionrWrXname)rjrpathjoinropenjsonload collections OrderedDictrr, write_modedumpwriteeolmkdirr+) rtrWrXruall_job_outputs_pathfdrajob_id job_homedirjob_queue_path job_queue job_entrys r"queue_entry_pointrs"77<< 3I(JL^_ "C (9B))B+:Q:QRs3#7889 "J /2"& /2a0   '',,rzz*@A6JKHH[WW\\"**-C"DFVWN nc ""bIIbM "%))+I  & Y nj )R )R*   M599"" Ms0=G 3G-G:(.H G*-G7:HHc tdtjjtjd|}tjj }tjjtjd||d<tjjtjdd}t |d5} tj| tj} dddt|tr|d|zd z|zd zz }n|d z|z}ttz|ztz} t|  t!| |g}t-|| t/| t tjj|dd5} tj0|| d| j3ddddtt5|tdztzt7dj9|ddk(rtjj|d} t | d5} tjj|d}t;j<||}t?|jAD]&\}}| j3djC||( dddttdztztDjDjG}|ddk(rdjCtIjJ|tIjJtjj|ddtIjJtjd|}gdtjjMd rd!gngz|gz}n1|dd"k(r)d#jCtO||d$%}||d&<d'd(|g}|}tQjR|)}|jUtDjDjG}|jVd*k7rEt%| d zt'd zd+zt+|jVzd,zt+||z ztjj|dd-}tjjY|r< t |d5} tj| tj}dddni}t| d1zt]zd2ztzd,zt+||z ztt5td3ztzt7d3j9t tjjtjddd5} tj| tj} ddd| |<| D]}| | t!| || d45t tjjtjddt^5} tj0| | d| j3t`dddy#1swYxYw#t"$r=} t%| d zt'd zd zt)t+| zYd} ~ d} ~ wwxYw#t"$r=} t%| d zt'd zdzt)t+| zYd} ~ d} ~ wwxYw#1swYdxYw#1swYvxYw#1swYExYw#t"$rJ} t%d.| zd/zt)t+| jZd0zt+| zzYd} ~ d} ~ wwxYw#1swYxYw#1swYyxYw)6a :param job_id: job ID of the local job to run :type job_id: string :param function: function to run :type function: string :param input_hash: input for the job (may include job-based object references) :type input_hash: dict :param run_spec: run specification from the dxapp.json of the app :type run_spec: dict Runs the specified entry point and retrieves the job's output, updating job_outputs.json (in $DX_TEST_JOB_HOMEDIRS) appropriately. z======rHOMErlrmrnNz (:) failedz when resolving input: z when processing depends_on: zjob_input.jsonrrprqrzInput: )title title_len interpreterbash environment)rz{}={} zLogs:a0 cd {homedir}; . {env_path}; . {code_path}; if [[ $(type -t {function}) == "function" ]]; then {function}; else echo "$0: Global scope execution complete. Not invoking entry point function {function} because it was not found" 1>&2; firh)homedirenv_path code_pathrt)r-cz-eDX_TEST_X_FLAGz-xz python2.7z#!/usr/bin/env python import os os.chdir({homedir}) {code} import dxpy, json if dxpy.utils.exec_utils.RUN_COUNT == 0: dxpy.run() code)rrDX_TEST_FUNCTIONpythonr)envrz, exited with error code z after zjob_output.jsonz Error: Could not load output of z: r]z -> zfinished runningzOutput: F)rG)1r^rrvrwrcopyrxryrzr{r|r(rrrrrNr@r#r rrrZrer~rr r,lstripr gen_bash_varsr)itemsformatdatetimenowshlexquoterrepr subprocessPopen communicate returncodeexists __class__rr}r)rrtrWrun_specrXrurjob_envrrrajob_namerJrjob_input_file var_defs_hashkeyval start_timescriptinvocation_argsr fn_processend_timejob_output_path job_output other_job_ids r"run_one_entry_pointrs (O'',,rzz*@A6JKjjooGggll2::.D#EvNGFO77<< 3I(JL^_ "C (SB))B+:Q:QRS$ # v #h.44|h&v%.H (Okz?;  J 3qJ8 bggll;(893 ?2 *b+  Jtv /ADF/J&))n 66IEQTXY\]^]h]hYilpYpsvwxsyYyTzz { { |SSs &Z Z 0 [)&*\2A:\? ]#&]  ] &^/.^<Z [&)2[!![&) \/22\**\/2\<?]  ]] ^,"?^''^,/^9<_c tjjtjdd}tjjtjdd} t |d5}t j |}dddtdk(ryt |d5}t j |}dddd}t|D]I\}}d} t|dr t|dt|dr| s8|j|}n||jd}t |t5}t j|||jtdddt!|d |d |d||j#d g|j#d N#1swY.xYw#1swYxYw#d } YxYw#1swYgxYw)a  :param run_spec: run specification from the dxapp.json of the app :type run_spec: dict Runs all job entry points found in $DX_TEST_JOB_HOMEDIRS/job_queue.json in a first-in, first-out manner until it is an empty array (or an error occurs). rrsrlTrmNrrWFrPrtrXru)rrtrWrrXru)rrvrwrrxryrzr, enumerater4rNr-r}r~rrrr) rrrrrraentry_point_to_runr3 entry_pointrunnables r"run_entry_pointsrsWW\\"**-C"DFVWN77<< 3I(JL^_  .# & &" " I & y>Q   & , ,"iimO ,"' 2 NA{H$[%>?*;|+DoV%[%>? %.]]1%5"   %"+q!1  .* -  IIi $ HHSM   #5d#;%7 %C'9,'G%-'9'='=lB'O!3!7!7!?  AE  & &  , ,$H  s00F2)F?+G ,G2F<?G  GG)Tr7)- __future__rrrrrrryrrr{rrArDrr r r r r printingrrrrrresolverrcompatrrr}rr#r4rLrNrUrZrerjrrrr8r$r"rs"SR''' BB55$     D')%5N +D +<O(z8:/bHT3Ar$