5ixddlmZddlZddlZddlZddlZddlZddlZddl m Z dZ dZ dZ dZdZd Zd d Zy) )print_functionN)file_load_utilsctj||D]5}tjtjj ||7y)a= Create a set of directories, so we could store the input files. For example, seq1 could be stored under: /in/seq1/NC_001122.fasta TODO: this call could fail, we need to report a reasonable error code Note that we create a directory for every file array, even if it has zero inputs. N)r ensure_dirospathjoin)idirdirsds /home/marpiech/ifpan-abm-pgxpred/analysis/marpiech-gwas-test/venv/lib/python3.12/site-packages/dxpy/bindings/download_all_inputs.py _create_dirsrs>t$ :""277<<a#89:c|d}tjj||d}td|zdz|ztj j tj|||S)N src_file_id trg_fnamezdownloading file: z to filesystem: ) rrr printsysstdoutflushdxpydownload_dxfile)file_recr src_filetrg_files r _download_one_filer,sc &Hww||D(;"78H  ),> > IJJJ8, Orc*|D]}t||yN)r) to_downloadr rs r _sequential_file_downloadr 6s+8T*+rc F tjj|5}|Dcic]}|jt|||}}tjj |D]}||} |j  dddycc}w#t$r-tjj|dd|ddwxYw#1swYyxYw#t$rAtdtjtjtjYywxYw)N) max_workersrz -> rz generated an exception)file) concurrentfuturesThreadPoolExecutorsubmitr as_completedresult ExceptionrstderrwriteKeyboardInterruptrr_exitEX_IOERR)rr max_num_parallel_downloadsexecutorr future_filesfutures r _parallel_file_downloadr5<s    2 2638 ;C,79 (%OO,>$OQYY9L9$,,99,G '/MMO   9 !JJ$$&.}&=x ?T&VW    bszz"  sX CC B )C 0BC C C 6CC  CCCAD D ctjtjd\}}i}d}|D]<}||vr||}||dz|d|||dz|d|||dz|d|>|S) z Create a dict of values for the downloaded files. This is similar to the variables created when running a bash app. Nc||vr|||<yyr)keyvaluedict_s r add_if_no_collisionz-_gen_helper_dict..add_if_no_collision]s e E#J r_pathr_namebasename_prefixprefix)ranalyze_bash_varsget_input_json_file)filtered_inputsfile_key_descs_ignoreflattened_dictr<input_input_var_dicts r _gen_helper_dictrJRs .??++-t5NGN"Z  ' '/FW,nV.DnUFW,nZ.H.YFY.x0H.Y Z rc Ht||tt|dz dS)zC Ensure at least ~1.2 GB memory per thread, see PTFM-18767 i)minmaxint) max_threads num_coresmem_available_mbs r _get_num_parallel_threadsrSls% {Is3/?/D+Eq'I JJrc$tj} tj}tj|\}}}g} |D]4} d} || | jdd|vrd} | s$| j| 6t|| |rtj||}g} |jD]} | j| |r~tjjdz }t!j"}t%|||}t&j(j+dj |t-| ||n t/| |t1|}|S#t$r#dj }|dz }t |wxYw) aS :param exclude: List of input variables that should not be downloaded. :type exclude: Array of strings :param parallel: Should we download multiple files in parallel? (default: False) :type filename: boolean :param max_threads: If parallel is True, how many threads should be used to download files? (default: 8) :type append: int :returns: dict of lists of strings where each key is the input variable and each list element is the full path to the file that has been downloaded. This function downloads all files that were supplied as inputs to the app. By convention, if an input parameter "FOO" has value {"$dnanexus_link": "file-xxxx"} and filename INPUT.TXT, then the linked file will be downloaded into the path: $HOME/in/FOO/INPUT.TXT If an input is an array of files, then all files will be placed into numbered subdirectories under a parent directory named for the input. For example, if the input key is FOO, and the inputs are {A, B, C}.vcf then, the directory structure will be: $HOME/in/FOO/0/A.vcf 1/B.vcf 2/C.vcf Zero padding is used to ensure argument order. For example, if there are 12 input files {A, B, C, D, E, F, G, H, I, J, K, L}.txt, the directory structure will be: $HOME/in/FOO/00/A.vcf ... 11/L.vcf This allows using shell globbing (FOO/*/*.vcf) to get all the files in the input order and prevents issues with files which have the same filename.z0Error: Could not find the input json file: {0}. zE This function should only be called from within a running job.T/rFz"Downloading files using {} threads)r get_input_dirrCget_job_input_filenamesIOErrorformatrsplitappendr filter_dictvaluesextendpsutilvirtual_memorytotalmultiprocessing cpu_countrSrr,r-r5r rJ)excludeparallelrPr job_input_filer inputsrestmsgdirs_to_creater keepr ival_list total_memrQr1 helper_varss r download_all_inputsrpssX  ( ( *D(<<>,DD^TfdN %  am Q7*   ! !! $ %~& ,,VW=K]]_& 9%&))+11R7 #--/ %>{IW`%a" =DDE_`a T3MN!+t4"6*K U AHHX VV c  s -E##,F)NF) __future__rconcurrent.futuresr%rrrcr`r dxpy.utilsrrrr r5rJrSrpr8rr rusD"&  &:"+ ,4KZr