5ijdZddlmZmZmZmZddlZddlZddlZddl Z ddl Z ddl m Z ddl mZddlmZddlmZdd lmZdd lmZhd Zej1hd Zej1hd ZGddeZdZdZdZdZ dZ!d,dZ"d,dZ#d,dZ$dZ%dZ&d-dZ'dZ(dZ)dZ*dZ+dZ,d Z-d!Z.d"Z/d.d#Z0d$Z1d%Z2d&Z3d'Z4d(Z5d)Z6d*Z7d+Z8y)/zz Workflow Builder Library +++++++++++++++++++ Contains utility methods useful for deploying workflows onto the platform. )print_functionunicode_literalsdivisionabsolute_importN)INTERACTIVE_CLI)process_extra_args)fill)json_load_raise_on_duplicates)err_exit)logger>titledetailssummary descriptiondeveloperNotes>nametagsdxapibillToversion categoriesregionalOptions>folderinputsstagesoutputsproject properties ignoreReuse outputFolderceZdZdZy)WorkflowBuilderExceptionzb This exception is raised by the methods in this module when workflow building fails. N)__name__ __module__ __qualname____doc__w/home/marpiech/ifpan-abm-pgxpred/analysis/marpiech-gwas-test/venv/lib/python3.12/site-packages/dxpy/workflow_builder.pyr#r#-s  r)r#c0tjj|s |jdj |tjj tjj ||stdj |ttjj ||5} t|cdddS#t$r*}tdj ||jd}~wwxYw#1swYyxYw)z Returns the parsed contents of a json specification. Raises WorkflowBuilderException (exit code 3) if this cannot be done. z{} is not a directoryz]Directory {} does not contain dxworkflow.json: not a valid DNAnexus workflow source directoryNz#Could not parse {} file as JSON: {}) ospathisdirerrorformatexistsjoinr#openr Exceptionargs)src_dirjson_file_nameparserdesces r*_fetch_spec_from_dxworkflowjsonr;5s 77== ! ,33G<= 77>>"'',,w? @& k VG_  bggll7N3 4q q06qq q*+P+W+WXfhihnhn+op p qqqs*D  C D %DD  D  Dcddl}|jddtj|j dd}tj |S)Nrz""\w*": (null|\{\}|""|\[\])(\,|)\s*z, }})resubjsondumpsreplaceloads) json_specr? clean_jsons r*_cleanup_empty_keysrGHs? @DJJyDYZbbchilmJ ::j !!r)c`d} ddlm}|j dru|dj drat |dj d}|jdd}||||krtd j|d |ytd #t$r ddlm}YwxYw) Nz2.8.0r) StrictVersionr)Versionrr-zjSource workflow {} is not compiled using dxCompiler (version>={}) that supports creating global workflows.rzCannot find the dxCompiler version from the dxworkflow.json/source workflow spec. Please specify it by updating the details field of the dxworkflow.json/source workflow spec using the 'version' key.) distutils.versionrI ImportError utils.versionrJgetstrsplitr#r0)rESUPPORTED_DXCOMPILER_VERSIONrJcompiler_version_usedcompiler_version_no_snapshots r*_check_dxcompiler_versionrUMs#* +>}}YIi$8$<$Q:RS23>>sCT?UV //0B>JDQbLc=d 0 f 223EAMtThOi@j 3 lr)c( |d}|d}||k(r||k(ry tjjd|dz|dddddi}|d|dk(xr |d|dk(S#tdxYw#tjj $rYy xYw) aK Returns True if a global workflow with the given name and version already exists in the platform and the user has developer rights to the workflow. "name" and "version" can be passed if we already made a "describe" API call on the global workflow and so know the requested name and version already exists. rrz}Both 'name' and 'version' fields must be given in the dxworkflow.json/source workflow spec to build/update a global workflow.Tglobalworkflow-fields)rraliasrxF)r#rbr{global_workflow_describe exceptions DXAPIError)rErrrequested_namerequested_version desc_outputs r*_version_existsrsh"6*%i0"3w"> ((;;idrinputrrexecutionPolicysystemRequirementsrzMWarning: the following stage fields are not supported and will be ignored: {}, )r#r0rurrlenr[r2)stage stage_indexrsunsupported_keyss r*_get_validated_stagersu5 & A H H UW W?N,UZZ\>J q  ]vdii 012 4 Lr)ct|ts tdt|}d|vrt |dk(ryt|Dcgc]}|j dc}}|D] }||vstdj |ycc}w)z Checks if each stage ID specified in ignore_reuse_stages exists in the workflow definition. If ignore_reuse_stages contains only '*', the field is valid. z9"IgnoreReuse must be a list of strings - stage IDs or "*"*rNrz]Stage with ID {} not found. Add a matching "id" for the stage you wish to set ignoreReuse for)ryrzr#r}rrOr0)rignore_reuse_stagesignore_reuse_setr stage_idsignoreds r*validate_ignore_reusers )4 0'(cd d./ 3'7#8A#=&9UYYt_9:I#A ) #*ovvw~A AA:sBct|ts tdg}t|D] \}}|j t ||"|S)zE Validates stages of the workflow as a list of dictionaries. z2Stages must be specified as a list of dictionaries)ryrzr# enumerateappendr)rvalidated_stagesindexrs r*_get_validated_stagesrsV fd #&'[\\!&)D u 4UE BCD r)ci}tjj|j\}}}t ||||d<t |||d<t ||}|s td|S||d<|S)zT Validates fields used only for building a regular, project-based workflow. rrz'Warning: workflow name is not specifiedr)rbexecutable_builderget_parsed_destination destinationrfrlror[)rEr5 validatedoverride_project_idoverride_folderoverride_workflow_namerns r*#_validate_json_for_regular_workflowrsI 66t7G7GHA*@3ItEXYIi1)_MIh&y2HIM  78 * & r)c d|vr tdtjjj |ds td|d|dj k7r't jdj|dd|vr tdtjjj |ds't jdj|dd|vrt|dts td d |vr\t|d tr=|d r8t|d jDcgc]}t|tc}s td y y cc}w) z Validates fields used for building a global workflow. Since building a global workflow is done after all the underlying workflows are built, which may be time-consuming, we validate as much as possible here. rzldxworkflow.json/source workflow spec contains no 'name' field, but it is required to build a global workflowz8The name of your workflow must match /^[a-zA-Z0-9._-]+$/z*workflow name "{}" should be all lowercaserzodxworkflow.json/source workflow spec contains no 'version' field, but it is required to build a global workflowz@"version" {} should be semver compliant (e.g. of the form X.Y.Z)rz(The field "details" must be a dictionaryrzXThe field "regionalOptions" must be a non-empty dictionary whose values are dictionariesN)r#rbrGLOBAL_EXEC_NAME_REmatchlowerr warnr0GLOBAL_EXEC_VERSION_RErydictallvalues)rEr5is r*"_validate_json_for_global_workflowrsv Y& z| |  " " 6 6 < /0i@Q6R6Y6Y6[\At,\]*jl l^&]sE,c$|sy|sytj|}t|jt}t |dkDr3t jdjdj|d|vrt|d|d<d|vr|jsvtjjtjj|j }||dk7r(t jdj|d|d|vrt#|d|dt%||}|j'||j(d k(rt+||}|j'||j(d k(r t-|||S) z Validates workflow spec and returns the json that can be sent with the /workflow/new API or /globalworkflow/new request. NrzRWarning: the following root level fields are not supported and will be ignored: {}rrrz;workflow name "{}" does not match containing directory "{}"r rYglobalworkflow)copydeepcopyrurrSUPPORTED_KEYSrr rr0r2r_fromr,r-basenamenormpathr6r'_get_validated_json_for_build_or_updateupdatemoderr)rEr5validated_specrdir_namevalidated_documentation_fieldsrs r*_get_validated_jsonr2sn  ]]9-N-^-@-@-BNS q  ` "234 6 >!#89Q#Rx   77##BGG$4$4T\\$BC ~f- - KKMTTUcdjUkmuv x&nX6}8UV%L^]a%b"89 yyJ7 4H i( yy$$*>4@ r)cztj|}|js*tjj ||j d|vrtjdd|vrtjd|S|djdr'tjdj|d|S)z Validates those fields that can be used when either building a new version (of a local, project-based workflow) or updating an existing version (of a global workflow). rz\dxworkflow.json/source workflow spec is missing a title, please add one in the 'title' fieldrz`dxworkflow.json/source workflow spec is missing a summary, please add one in the 'summary' field.z:summary {} should be a short phrase not ending in a period) rrrrbrinline_documentation_filesr6r rrir0)rEr5rs r*rras  i(I :: ::9dllSi rs ! vw  Y  ( ( - KKT[[\efo\pr s r)c v|s|St|jdDcgc]}|jdc}}|D]}|jdrotj|j }tj |j}|h|k7sZtdj|dj||jdrtjj|ddd ii }t|dj}|j|r|j!|} d j|d j|d j|} | d jd j| z } | djd j| z } t#j$| |j'||jdr-tjj)|} t+|| }|jdstd|Scc}w)aa Check if the dependent apps/applets/subworkflows in the workflow are enabled in requested regions Returns the subset of requested regions where all the dependent executables are enabled If the workflow contains any applets, then the workflow can be currently enabled in only one region - the region in which the applets are stored. rrzapplet-z:AAM"::./m06uchhG_>`0acc  f %#'88#8#8hYjlpXqMr#8#s 23DEJJLMK+44[A.F.Q.QR].^+]dd499[1499=U3VXFMMdiiXsNtuuBIIII9:<< D!(<<[I   k *"&(("<"tDL)5 v & LLCJJ<X Y    !''AABTB[B[B]^ Js AA;Bc4t||d}i} |jD]:\}}||d<d|d<t|}tjd|zdz|||<< ||fS#|r-t j j|jxYw)z Creates a workflow in a temporary project for each enabled region. Returns a tuple of dictionaries: workflow IDs by region and project IDs by region. The caller is responsible for destroying the projects if this method returns properly. rrrhrzCreated workflow z successfully) ritemsrr rrbrrr)rrEr5rworkflows_by_regionrrrs r*_build_underlying_workflowsrs 4OYxEXY 1779 6OFG$+Ii "%Ih 1)P>W>W>Y Z s A A%%2Bc  ii}} |jdi}|rt|jt|k7rJtddj t|j t|jt j|}ntj|i}t|||\}}|jD] \}}|||d<|jd|itjt|j t fd|jD} t j| } d| vr| d=d| vr| d=tj dj#t%j&| t(j*j-| d } tj d j#| d | d  tj dtj dj#| d | d  |r.t(j.j1|j3 t5| | jdg| S#|r.t(j.j1|j3wwxYw#tj6dj#| d | d  xYw)z Creates a workflow in a temporary project for each enabled region and builds a global workflow on the platform based on these workflows. rzThese enabled regions do have regional options specified in the JSON spec or from --extra-args: {}rrYc36K|]\}}|vs ||fywrqr().0kvgwf_provided_keyss r* z)_build_global_workflow..0s!]AaK\F\q!f]s  rrz)Will create global workflow with spec: {}rz-Uploaded global workflow {n}/{v} successfullyrr)nr z#You can publish this workflow with:z dx publish {n}/{v}rz>The workflow {n}/{v} was created but setting categories failed)rOr}rrr#r2r~rrrfromkeysrrrGLOBALWF_SUPPORTED_KEYS intersectionr infor0rArBrbr{global_workflow_newrrrrr) rErr5rrexisting_regional_optionsupdated_regional_optionsrrgwf_final_json print_specrr s @r*_build_global_workflowrs /1"+-[$-MM2CR$H! $,1134O8LL.0V+.88C4H4S4STWXqXvXvXxTy4z+{}}'+mm4M&N $'+}}_b'I $ (D I 0/#6#<#<#> G FK;F $V ,Z 8 G +-EFG4@@Y^^EUAVW]1B]]]]>2 N *=) ~ -+, ?FFtzzR\G]^_"XX99.I$O CJJ^\bMcM[\eMfKh i 9: *11N64J4B94M2O P   # # = =>P>W>W>Y Z#$68J8JP>W>W>Y Z   L S SVdekVlVdenVo T q r sH)I?!J4?2J14/K#cbfd}tdjD}t|}td|jDjr7|s0tj j ddzdd}|Stjd |S) Ncd}rdjdddj}jr{tr\ t dt t d|zt dt d}|jjd s2d }|Stjd d }|Stj||Sd }tjd |S#t$rd }YywxYw)NFz]The global workflow {}/{} exists so we will update the following fields for this version: {}.rrrz***zINFO: z$Confirm making these updates [y/N]: ryTzskipping requested change to update a global workflow version. Rerun "dx build" interactively or pass --yes to confirm this change.zNothing to update) r0r2confirmrr[r rKeyboardInterruptrrr rr) skip_updateupdate_messagevaluer5rEnon_empty_fieldss r*rz,_update_global_workflow..skip_updateUs |DD&!9Y#7CS9TVN||"$e d8n#<=>e %&L M!;;=33C8&* KK^_"&K   N+K KK+ ,-$ #$s8C%% C32C3c3<K|]\}}|tvs||fywrq)UPDATABLE_GLOBALWF_FIELDSr r r s r*r z*_update_global_workflow..rs ^$!QqD]?]1v^s c30K|]\}}|s ||fywrqr(r$s r*r z*_update_global_workflow..tsKtq!QFKs  rrrrrzSkipping making updates) rrrrrbr{global_workflow_updater r)rEr5rr update_specrr!s`` @r*_update_global_workflowr(Ss:^)//*;^^K<[$ONK~/C/C/EKK {{;=!XX<<=NQZ[aQb=bCLYCWJZ=\\`b   -. r)ct |jdk(rZt|jd|}|j|jxsit ||}t ||j}|S|jdk(rt|jr |j}nt|jd|}|jr|j|d<|j|jxsit ||}|jdr4d|dvr-t||jst|t|t|}t j"j%d|dz}|r:t'||j(|j*rt-|||j.}|St j"j1|j2t4|d <t7||}t9|||} |St5d j;|j#t j<j>$r}|d }~wwxYw) z Creates or updates a workflow on the platform. Returns the workflow ID, or None if the workflow cannot be created. rYzdxworkflow.jsonrrr dxCompilerrrrzUnrecognized workflow type: {}N) rr;r6r extra_argsrrrrversion_overriderOrUbriefr]r`rGrbrverify_developer_rightsrrrr(rget_valid_bill_torr#rrr0rr)r5r8rErexisting_workflowrr:s r*_build_or_update_workflowr1s . 99 "7 FWY_`I   T__2 3+It(3+I6I !% 7 7 O OPadmntduPu v  _Y5F5K5K5F5N5N&P6iGXG[G[\  '+&=&=&O&OPTP\P\^v&w (#"@D"Q4YQUV ++K+R+RSWS\S\+]^ ^ ?? % %s,A'H+D5H"A H/$HH70H22H7c:|jr/|jdk(r tjj |}n?|jr/|jdk(r tjj |}nd|i}|t tj|yy)NrYrr)rArrbr{rrr[rB)rr5outputs r* _print_outputr4su yyTYY*,++K8 tyy$4422;? $  djj !r)c| td t|t||}t||y#t$rI}t dj |tjtjdYd}~yd}~wwxYw)z Validates workflow source directory and creates a new (global) workflow based on it. Raises: WorkflowBuilderException if the workflow cannot be created. NzArguments not providedz Error: {})file) r4r r1r4r#r[r0sysstderrexit)r5r8rr:s r*buildr;sn  |0114 /f= k4( # k  ##**5  s#3 B?BBrq)NN)F)9r' __future__rrrrr,r8rArrbclir cli.parsersr utils.printingr utilsr rr r=r r#unionrrr4r#r;rGrUr]r`rfrlrorurrrrrrrrrrrrrrrr(r1r4r;r(r)r*rBs $SR + 0 \399;de(..0bc y q&" q33.&  2>l$:"A( $lF,^.*$X(V<36>B)X4n"r)