5ioxddlmZmZmZmZmZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlmZddlmZddlmZddlmZmZddlmZdd lmZmZmZmZmZdd l m!Z!dd l"m#Z#dd l$m%Z%m&Z&m'Z'm(Z(m)Z)dd l*m+Z+m,Z,ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9ddl:m;Z;ddlm?Z?ddl@mAZAmBZBddlCmDZDddlEmFZFddlGmHZHddlImJZJddlKmLZLddlMmNZNmOZOmPZPddlQmRZRddlSmTZTddlUmVZVddlWmXZXddlYmZZZm[Z[ddl\m]Z]m^Z^e jd Z`e jd!Zad"Zbd#Zcd$Zdd%ZedFd&Zfd'Zgd(Zhd)Zid*Zjd+Zkd,ZldGd-ZmdHd.Znd/Zod0Zpd1Zqd2d3d4ggerd5e jfd6Ztd7Zud8Zvd9ZwGd:d;exZyd<Zzd=Z{dId>Z|d?Z}Gd@dAeZ~GdBdCZGdDdEZy)J)print_functionunicode_literalsdivisionabsolute_import annotationsN)reduce)fill)DXRecord) is_dxlinkdescribe)DXFile)resolve_existing_path is_hashidResolutionError resolve_pathcheck_folder_exists) as_handle) print_desc)err_exitPermissionDenied InvalidInput InvalidStatedefault_expected_exceptions) validate_JSON final_payload) get_genotype_only_typesadd_germline_base_sqlsort_germline_variantharmonize_germline_sqlharmonize_germline_resultsget_germline_ref_payloadget_germline_loci_payloadupdate_genotype_only_refget_genotype_typesinfer_genotype_type%get_types_to_filter_out_when_inferingfilter_results)inference_validation)validate_somatic_filter)somatic_final_payload)cohort_filter_payloadcohort_final_payload)Dataset) ArgsValidator) JSONValidator)*EXTRACT_ASSAY_EXPRESSION_INPUT_ARGS_SCHEMA)$EXTRACT_ASSAY_EXPRESSION_JSON_SCHEMA)1EXTRACT_ASSAY_EXPRESSION_FILTERING_CONDITIONS_1_01EXTRACT_ASSAY_EXPRESSION_FILTERING_CONDITIONS_1_1?EXTRACT_ASSAY_EXPRESSION_FILTERING_CONDITIONS_1_1_non_optimized)JSONFiltersValidator)VizPayloadBuilder) VizClient)transform_to_expression_matrix)write_expression_outputpretty_print_json)"EXTRACT_ASSAY_EXPRESSION_JSON_HELP/EXTRACT_ASSAY_EXPRESSION_ADDITIONAL_FIELDS_HELPz^database_\w{24}__\w+$z^database-\w{24}$ct|\}}}|td|zdz||ddk7rtd|zdz|zdz|dddk7rtd j| t j d |d zd z|d d}ddk7rtd|dzd|dvsd|dvr|d}ntd|dz|||fS#t $rtdt fYattf$r td|d zttfYt$r}tt|Yd}~d}~wwxYw)NzUnable to resolve "z." to a data object or folder name in a projectr projectz&" to a data object or folder name in ''classrecordzK{}: Invalid path. The path must point to a record type of cohort or dataset/idz /visualizeF)r@ cohortBrowserzInsufficient permissionsexpected_exceptions%s : Invalid cohort or datasetdatasetVersionz3.0z>%s : Invalid version of cohort or dataset. Version must be 3.0r. recordTypes CohortBrowserdatasetRecordProjectzL%s : Invalid path. The path must point to a record type of cohort or dataset) rrrformatdxpy DXHTTPRequestrrr Exceptionstr)pathr@ folder_path entity_resultrespdetailsdataset_projects |/home/marpiech/ifpan-abm-pgxpred/analysis/marpiech-gwas-test/venv/lib/python3.12/site-packages/dxpy/cli/dataset_utilities.pyresolve_validate_record_pathrZYs*?*E'G[- ! > ?  M*-i8 8 ! 7 8     Z )X5 Y ` `ae f !! -% % 4% 8 " & L#$ % T-((omAT.T56 Z=! " M4 887 V+BRATU , '  ,}T/B B!  Ws$+"CE.+EE#D<<Ec|ddz|dzdz|z} tj||d}ddz}|S#t$r}tt |Yd}~,d}~wwxYw) Nurlz/viz-query/3.0/datasetrDFresourcedata prepend_srvsql;)rOrPrQrrR)rVpayloadroute resource_val resp_queryrW sql_resultss rYviz_query_api_callrisx;!22T)_DsJURL''!U U#c)K  Ws8 AAAct||dS)Nz raw-queryrirVrds rYraw_query_api_callrms dG[ 99ct||dS)Nzraw-cohort-queryrkrls rYraw_cohort_query_api_callrps dG-? @@rnc|ddz|dzdz} tj||d}d|jvr]|ddd k(r d |dd z}n2|r|ddd k(r d |dd z}n|dddk(r |dd }n|d}tt ||S#t $r}tt |Yd}~Sd}~wwxYw)Nr\z /data/3.0/r]z/rawFr^errortyperz4Insufficient permissions due to the project policy. message QueryTimeOutzhPlease consider using `--sql` option to generate the SQL query and query via a private compute cluster. DxApiError)rOrPkeysrrRrQ)rVrd sql_messagerfresp_raw err_messagerWs rY raw_api_callr{s;-Y?&HL%%!U  hmmo % (N:UX`ahXijsXtt '!26!:n!LJMUV]M^_hMii '"6*l:&w/ : &w/ S% & O W OsBB C%B??Cc |js;|js/|js#|j|j t d|j|j t dddddddd}d}|jr*d|d<|||}|rt d j ||jr/d|d <d|d <|||}|rt d j |ttj|jd }t|dk(r|dk7r|dk(rd}n|dk(rd}nd}n t dt|j\}}}} |d} d} d} d} g}g}|jrtjj dk(rtjj"dk(rdnd} ddlat)t+t,t&j.j1ddd}|dkr*t3dj |t&j.||j6t9j:} n|j6d!k(rd"} nvt8jj=|j6rBt8jj?|j6r |j6} nt d#n t d#| r1tj@}tj@}tj@}nt8jjC| |d$d%zz}t8jjC| |d$d&z|z}t8jjC| |d$d'z|z}|||g}|js |jr|jDrd(}n}|j6Mt9j:} t8jjC| |d$|z} |jG| n|j6d!k(rd"} npt8jj=|j6rt8jj?|j6rD|j6} t8jjC| |d$|z} |jG| n|jG|j6nt8jj=t8jjI|j6s)t8jjI|j6s |j6} nBt d)j t8jjI|j6*|D]3}t8jj=|s#|jG|5|rt d+j |*tK| | ,jM}g}|j7|jj1dDcgc]}|jO}}n|jt8jjQ|jrHtS|jd-5}|D]"}|jG|jOd.$ dddn%t d/j |j0|ryg}|D]}|j1d}t|dkr|jG|4|d|jTd jWvs,|d|jTd |dd1jWvs|jG||rt d2|z||Dcgc]$}|d3jC|j1di&c}d4} d5|d6vr-|jYd7r|jYd7| d8<|d9| d9<|jDr=t[|| }!| r t3|!nZtS| d:5}"t3|!|";dddn7t]|| }#t_| | ||#d<|=n|jDr t d>|jr%|ja}$|$jc|?}%|js |jrcte|jT\}&}'|jrt3d.jC|&|jrt|jT|'|yyy#t4$r&}t d j ||Yd}~#d}~wwxYwcc}w#1swYxYwcc}w#1swYxYw)@z Retrieves the data or generates SQL to retrieve the data from a dataset or cohort for a set of entity.fields. Additionally, the dataset's dictionary can be extracted independently or in conjunction with data. NzMust provide at least one of the following options: --fields, --fields-file, --dump-dataset-dictionary, --list-fields, --list-entitieszpdx extract_dataset: error: only one of the arguments, --fields or --fields-file, may be supplied at a given timeF,)dump_dataset_dictionaryrbfields fields_fileoutputdelimc g}|jD]H\}}|j||k7s|jdj|j ddJ|S)Nz--{}_-)items__dict__appendrNreplace)args restricted error_listoptionvalues rY check_optionsz&extract_dataset..check_optionss` '--/ KMFE}}V$-!!&--sC0H"IJ Krn list_entitiesz*--list-fields cannot be specified with: {} list_fieldsentitiesz,--list-entities cannot be specified with: {}unicode_escaper:"z.csv .tsvz.txtzInvalid delimiter specifiedr]z2.0.3z2.2.3r.r )r rzWarning: For '-ddd' usage, the recommended pandas version is {}. The installed version of pandas is {}. It is recommended to update pandas. For example, 'python3 -m pip install -I pandas=={}'.z'-ddd' requires the use of pandas, which is not currently installed. Please install pandas to a version {}. For example, 'python3 -m pip install -I pandas==X.X.X' where X.X.X is {}.rTz>Error: When using -ddd, --output must be an existing directory recordNamez.data_dictionaryz.codingsz.entity_dictionary .data.sql Error: {path} could not be foundrSz!Error: path already exists {path}r@r zNThe file, {input_fields_file}, supplied using --fields-file could not be found)input_fields_filerz(The following fields cannot be found: %s$)project_contextrrLrKbaseSqlbase_sqlfilterswfileresults out_file_nameprint_to_stdoutsep raw_results column_namesz3`--sql` passed without `--fields` or `--fields-file)output_file_dataoutput_file_entityoutput_file_codingr)3r~rrrrrrNrRcodecsdecoderlenrZrSsys version_infomajorminorpandaspdtuplemapint __version__splitprint ImportErrorrosgetcwdexistsisdirstdoutjoinrbrdirname DXDatasetget_descriptorstripisfileopenmodelrwgetrmr{ csv_from_jsonget_dictionarywriteretrieve_entities)(rlisting_restrictedrr delimiter out_extensionr@rUrVrX dataset_id out_directoryout_file_fieldrfiles_to_checkfile_already_existrecommended_pandas_versionsystem_pandas_versionerrrfile_name_suffixrrec_descriptor fields_listfieldinfilelineentry entity_fielditemrdrhfryrec_dictwrite_otentity_names_and_titles _main_entitys( rYextract_datasetrsw  ( (  "" KK     $ U     ( ~ $)  .3?+"4);<  AHHT U ,1=))-:&"4);<  CJJ:V WFMM$**.>?@I 9~yC/  "M $ "M"M./4PQUQZQZ4[1G]D/iJMNON ##140@0@0F0F!0KPSP`P`PfPfjkPkWqx"  $)#c2>>3G3G3LRa3P*Q$R !%v-KKQ62BNND^L ;; IIKM [[C "O WW^^DKK (ww}}T[[) $ T U V "zz !$ !$ !ww||tL14FFV  "$tL1J>N" "$tL14HH=X" /0BDVWN {{d&& 88* ,  ;; IIKMWW\\tL14DDN  ! !. 1 [[C "O WW^^DKK (ww}}T[[) $ !#!4 #58H#H"%%n5"))$++6 WW^^BGGOODKK8 9 KKB "[[N 2995:  , 77>>$   % %d +,4;;AS;TUz?CRRTNK {{26++2C2CC2HIu{{}I I   % 77>>$** +d&&, 9"9D&&tzz$'789 9 9 `gg&*&6&6h    )E ;;s+L< 1$!!%(Q~';';J'G'L'L'NN?%++J7 QH$& !!%( )  ?*L M 'EPQTchhtzz#78Q  d=1 1xx "&*hhy&9 #!%iGI  88,T7;Kk".#./!+A.//$D'2H , /$Y/(   FG ##!002>>-11 "  T--0A.BVBV0W-    $))34 5    ,,lD A .u  CDJDJ..D   hJ 9 9<R//s=7A,d e(e )e !e% e %ee e%e.c|j}g}g}|std||fS|D]-}|d|k(r|j||j|/||fS)NzNo valid assays in the dataset.generalized_assay_model)assaysrr)r assay_type assay_listselected_type_assays other_assaysas rYget_assay_inforsy&&JL 23 !, //  'A*+z9$++A.##A&  ' !, //rncd|z}tt|d|z}i}|jdrtjj |rjtj |jdk(rtdj|nt|d5} tj|}dddnStd j| n7|d k(rtdj|n tj|}|d vrt!|||S|dvr t#|||S#t$r}tdt Yd}~d}~wwxYw#1swYTxYw#t$r}tdt Yd}~vd}~wwxYw)Nzargs.retrieve_ retrieve_z.jsonrz~No filter given for --retrieve-{filter_type} or JSON for "--retrieve-{filter_type}" does not contain valid filter information.) filter_typerz)JSON for variant filters is malformatted.rGz/JSON file {filter_json} provided does not exist) filter_jsonz{})allele annotationgenotype)variant)rRvarsendswithrrSrstatst_sizerrNrjsonloadrQrloadsrr*)rr filter_arg filter_valuefilter json_file json_errors rYjson_validation_functionrs!K/JtDz+ ";<=L FW% 77>>, 'ww|$,,1U\\$/\ ,, !%9!5 AHH ,I  4  QXX +X   L1::fk* M  # 4 MA% G0K0 ?(C sHED3:E&3 E<E EEEE#& F /FF c d}gd}||Dcgc]}|dj||fic}d||dd} t|| d} t||d| d | ycc}w) Nvcf_meta_information_unique)FieldIDTypeNumber DescriptionrFnamerE)rr is_cohortvariant_browser)rxrrr)rr{r) rV project_idassay_id assay_namerrtablecolumnscolumnrdrys rYretrieve_meta_infor s| )E>G%>E 4:VSXXufo. /  GD'u=H#' Y'  sAc0|jrd}n|dk(r|jrd}nd}g}g}d}d}|jJtj}tj j |||z}|j|nK|jdk(rd}n8tj j|jrQtj j|jr td n|j|jntj jtj j|js)tj j|js |j}nBtd jtj j|j |D]3} tj j| s#|j| 5|r td ||fS) Nrsomaticz.vcf_meta_info.txtrrFrTz+--output should be a file, not a directory.rrz1Cannot specify the output to be an existing file.) rbr rrrrSrrrrrrrN) r record_namefriendly_assay_typerrrout_filerrrs rYassign_output_methodr&s xx&  )d.E.E/!NHO {{ 77<< {=M/MNh'    $ 77== % B C  % %dkk 2  4 5RWW__ >;; . 5 5277??4;;;W 5 X , 77>>$   % %d +,DE _ $$rnc|dk(rd}n|dk(rd}|rWt|\}}|stdj|n*|D]}t|dt j dt|\}}|D cgc]} | d } } |D cgc]} | d  } } |D cgc]} | d } } | r | r | d}| d}ntdj||ri|t | vrE|t | vrtd j|n4td j|| n|}|D]} | d|k(s | d }i}|dk(r^d }|D]V}|dk(s |dr|ddjddd}|dddddd|d<dD]}||vs||||<Xn1|dk(r,d}|D]%}|dk(s |ds|dddz|ddz}'|fScc} wcc} wcc} w)zN Generalized function for determining assay name and reference genome germlinegenetic_variantr"somatic_variant)rz,There's no {} assay in the dataset provided.rruuidz|This is not a valid assay. For valid assays accepted by the function, `extract_assay {}`, please use the --list-assays flag.z0Assay {assay_name} does not exist in the {path}.)rrSz GRCh38.92reference_genomerr:rrrrsmappingrgenotype_type_table)exclude_refdataexclude_halfrefexclude_nocallr referenceannotation_source_version)rrrNrrexitlistr) list_assaysrrSr$rr target_assaysrrgatarget_assay_namestarget_assay_idsoaother_assay_namesselected_assay_nameselected_assay_idadditional_descriptor_infoselected_ref_genomeexclude_genotypes rYget_assay_name_inforBCs j(&  )& (6 z) %  CJJK^_ `" !ai  ! HHQK%3:%!]L0=="V*==-:;r6 ;;.:;F;;.03,Q/?FFGZ[\ T"45 5T"344SZZ+ FMM#-DN #- # 3f:+(*6 % 3"$j() [Ay//'(*+,>*?*G*M*McST*UVW*X'DEjMR\D]^fDghnDopyDz|CED*+@A(`[$'1,GHIYGZ23CD[  [  )  qAy//AkN&' nV&;;s= G G$! G)c |jddt|z }tjd|t |fd|i|tj S)Nwidth_adjustmentr^)flags)poprresubr MULTILINE)stringcomment_stringkwargsrDs rY comment_fillrNsKzz"4a83~;NN 66#~tF'`EU'`Y_'`hjhtht uurncn|dddigd||did}t||dDcgc]}|d c}Scc}w)zR Get the list of sample_ids from the sample table for the selected assay. rM sample_idzsample$sample_id assay_filtersr)rr raw_filtersr)r{)rVrrsample_payloadrs rYretrieve_samplesrTsS  67!345'*H)MNN %1~$Fy$Q RqAkN RR Rs 2c"d}|js|js |jrd}|jrI|s t dn;|j s$|j s|js |jr t d|j r=|jr t dn%|jr t dn |r t d|js |jr;|js$|js|js |j r t d|jr|jrttd d d ztd zd ztdzd ztdzd ztdzd ztdzd ztdd zdzdztjdn9|jr|jr ttd d d ztdzd ztdzd ztdzd ztdzd ztdzd ztdzd ztdzd ztdzd ztdd zdzdztjdnd|jrW|jrJttd d d ztd zd ztd!zd ztd"zd ztd#zd ztd$zdztd%zdztd&zdztd'zdztd(zdztd)zdztd*zdztd+zd ztd,d zd ztd-d zdztd.d/gd0d1d2d3d4d2ggd5d6zdztd7d zdztd.d/gd8d9ggd:d;ztjd|jr t!d<|}n1|jr t!d=|}n|jr t!d>|}|j r |s t d?|jr |s t d@t#|j$\}}}}dA|dBvr,t'|j |j gr t dC|dD}t)||Ej+}t-|j |j |j$dF|\} } } } t/||dGdF\} }dH}|jrd<}n|jrd=}|r|rt1| | || |I\}}t3|||jr=t5||}|r t|n]t7| dJ5}t||KdHdHdHn:t9||}t;|dLt<M}t?| |dN||tAdOP|jr|r| jCdQ}| jCdR}| jCdS}tE|j|j||||js |jr tG|jCdTg}g|dT<dU|vrg}ntI||||}t1|| | || d>| V\}}t3||g}|rtK|D]\}}|jM}|dWk(rd|dX<n|dYk(rd|dZ<n |d[k(rd|d\<t1|| | || d]|dWk7|dYk7|d[k7|tO|d^z k(_ \} }!t3|| |jQ| tS|}"|jrg}#|"rPt5||dHd`}$ da| dbzdcz}%tUjV|%|$jY|#jQ|$|D]A} | dfjQdWdgit5|| dHd`}&|#jQta|&Cdhjc|#diz}|r t|yHt7| dJ5}t||KdHdHdHyHg}|"r t9||}'|je|'dL|D],} t9|| }(|jetg|(dL|.|r&ti||})|)rt9||)}*tk||*|js |jrtm|| | }+to|jCdjg},|,rtq|,js|+}+tu|dk|}-t9||-dLD.cgc]}.|.}/}.|jrdWnd[}0tw|+|/||0}tOdkDr ty|dT|}|j{t<Mt?| |dN||tAdOPyHyHyH#1swYxYw#tZ$r(t ddj]| dbt^feYIwxYw#1swYyHxYwcc}.w)lz Retrieve the selected data or generate SQL to retrieve the data from an genetic variant assay in a dataset or cohort based on provided rules. FTzPlease specify one of the following: --retrieve-allele, --retrieve-genotype or --retrieve-annotation" for details on the corresponding JSON template and filter definition.zPlease check to make sure the parameters are set properly. --json-help cannot be specified with options other than --retrieve-annotation/--retrieve-allele/--retrieve-genotype.z3The flag, --sql, cannot be used with --list-assays.zUWhen --list-assays is specified, output is to STDOUT. "--output" may not be supplied.5--list-assays cannot be presented with other options.zUThe flags, --infer-ref and --infer-nocall, can only be used with --retrieve-genotype."Filters and respective definitions# rL # a?rsid: rsID associated with an allele or set of alleles. If multiple values are provided, the conditional search will be, "OR." For example, ["rs1111", "rs2222"], will search for alleles which match either "rs1111" or "rs2222". String match is case sensitive. Duplicate values are permitted and will be handled silently.atype: Type of allele. Accepted values are "SNP", "Ins", "Del", "Mixed". If multiple values are provided, the conditional search will be, "OR." For example, ["SNP", "Ins"], will search for variants which match either "SNP" or "Ins". String match is case sensitive.aUdataset_alt_af: Dataset alternate allele frequency, a json object with empty content or two sets of key/value pair: {min: 0.1, max:0.5}. Accepted numeric value for each key is between and including 0 and 1. If a user does not want to apply this filter but still wants this information in the output, an empty json object should be provided.a<gnomad_alt_af: gnomAD alternate allele frequency. a json object with empty content or two sets of key/value pair: {min: 0.1, max:0.5}. Accepted value for each key is between 0 and 1. If a user does not want to apply this filter but still wants this information in the output, an empty json object should be provided.zlocation: Genomic range in the reference genome where the starting position of alleles fall into. If multiple values are provided in the list, the conditional search will be, "OR." String match is case sensitive.z*JSON filter template for --retrieve-allelera{ "rsid": ["rs11111", "rs22222"], "type": ["SNP", "Del", "Ins"], "dataset_alt_af": {"min": 0.001, "max": 0.05}, "gnomad_alt_af": {"min": 0.001, "max": 0.05}, "location": [ { "chromosome": "1", "starting_position": "10000", "ending_position": "20000" }, { "chromosome": "X", "starting_position": "500", "ending_position": "1700" } ] }ra^allele_id: ID of an allele for which annotations should be returned. If multiple values are provided, annotations for any alleles that match one of the values specified will be listed. For example, ["1_1000_A_T", "1_1010_C_T"], will search for annotations of alleles which match either "1_1000_A_T" or ""1_1010_C_T". String match is case insensitive.a+gene_name: Gene name of the annotation. A list of gene names whose annotations will be returned. If multiple values are provided, the conditional search will be, "OR." For example, ["BRCA2", "ASPM"], will search for annotations which match either "BRCA2" or "ASPM". String match is case insensitive.a'gene_id: Ensembl gene ID (ENSG) of the annotation. If multiple values are provided, the conditional search will be, "OR." For example, ["ENSG00000302118", "ENSG00004000504"], will search for annotations which match either "ENSG00000302118" or "ENSG00004000504". String match is case insensitive.afeature_id: Ensembl feature id (ENST) where the range overlaps with the variant. Currently, only coding transcript IDs are searched. If multiple values are provided, the conditional search will be, "OR." For example, ["ENST00000302118.5", "ENST00004000504.1"], will search for annotations which match either "ENST00000302118.5" or "ENST00004000504.1". String match is case insensitive.awconsequences: Consequence as recorded in the annotation. If multiple values are provided, the conditional search will be, "OR." For example, ["5_prime_UTR_variant", "3_prime_UTR_variant"], will search for annotations which match either "5 prime UTR variant" or "3 prime UTR variant". String match is case sensitive. For all supported consequences terms, please refer to snpeff: http://pcingola.github.io/SnpEff/se_inputoutput/#effect-prediction-details (Effect Seq. Ontology column). This filter cannot be specified by itself, and must be included with at least one of the following filters: "gene_id", "gene_name",or "feature_id".a[putative_impact: Putative impact as recorded in the annotation. Possible values are [ "HIGH", "MODERATE", "LOW", "MODIFIER"]. If multiple values are provided, the conditional search will be, "OR." For example, ["MODIFIER", "HIGH"], will search for annotations which match either "MODIFIER" or "HIGH". String match is case insensitive. For all supported terms, please refer to snpeff: http://pcingola.github.io/SnpEff/se_inputoutput/#impact-prediction. This filter cannot be specified by itself, and must be included with at least one of the following filters: "gene_id", "gene_name", or "transcript_id".ahgvs_c: HGVS (DNA) code of the annotation. If multiple values are provided, the conditional search will be, "OR." For example, ["c.-49A>G", "c.-20T>G"], will search for annotations which match either "c.-49A>G" or "c.-20T>G". String match is case sensitive.a hgvs_p: HGVS (Protein) code of the annotation. If multiple values are provided, the conditional search will be, "OR." For example, ["p.Gly2Asp", "p.Aps2Gly"], will search for annotations which match either "p.Gly2Asp" or "p.Aps2Gly". String match is case sensitive.z.JSON filter template for --retrieve-annotationa{ "allele_id":["1_1000_A_T","2_1000_G_C"], "gene_name": ["BRCA2"], "gene_id": ["ENST00000302118"], "feature_id": ["ENST00000302118.5"], "consequences": ["5 prime UTR variant"], "putative_impact": ["MODIFIER"], "hgvs_c": ["c.-49A>G"], "hgvs_p": ["p.Gly2Asp"] }aallele_id: ID(s) of one or more alleles for which sample genotypes will be returned. If multiple values are provided, any samples having at least one allele that match any of the values specified will be listed. For example, ["1_1000_A_T", "1_1010_C_T"], will search for samples with at least one allele matching either "1_1000_A_T" or "1_1010_C_T". String match is case insensitive.zlocation: Genomic position in the reference genome of the starting position of the alleles. If multiple values are provided in the list, the conditional search will be, "OR." String match is case sensitive.z6allele_id and location are mutually exclusive filters.asample_id: Optional, one or more sample IDs for which sample genotypes will be returned. If the provided object is a cohort, this further intersects the sample ids. If a user has a list of samples more than 1,000, it is recommended to use a cohort id containing all the samples.zegenotype_type: Optional, one or more genotype types for which sample genotype types will be returned.zOne of:z5 ref (homozygous for the reference allele e.g. 0/0)zC het-ref (heterozygous for the ref allele and alt allele e.g. 0/1)z3 hom (homozygous for the non-ref allele e.g. 1/1)z? het-alt (heterozygous with two distinct alt alleles e.g. 1/2)zC half (only one allele is known, second allele is unknown e.g. ./1)z/ no-call (both alleles are unknown e.g. ./.)z-JSON filter templates for --retrieve-genotypezExample using location:s1s2110000) chromosomestarting_positionX500)refhet-refhomhet-althalfno-call)rPlocation genotype_typezExample using allele_id: 1_1000_A_T 2_1000_G_C)rdrerfrg)rP allele_idrjrrrzc--assay-name must be used with one of --retrieve-allele,--retrieve-annotation, --retrieve-genotype.zWhen --sql provided, must also provide at least one of the three options: --retrieve-allele ; --retrieve-genotype ; --retrieve-annotation .rLrKCurrently --assay-name and --list-assays may not be used with a CohortBrowser record (Cohort Object) as input. To select a specific assay or to list assays, please use a Dataset Object as input.r]rr(rN)full_input_dictrrErgenome_referencerrrr)keyr|)rrrrr quote_charr/r0r1rjrm)rorrErrprorderrcref_ynrg halfref_ynrh nocall_yn genotype_onlyr:) rorrErrprr/r0r1rtz\br.z\w+z2Failed to find the table, {}, in the generated SQLrGrz allele$refz UNION rcrPri)>retrieve_alleleretrieve_annotationretrieve_genotype json_helprr6rrbr infer_ref infer_nocallrrNrr4r<rrZrSanyrrrBr&rrrmrr{sortedrrrRrr)r'r enumeratecopyrrr%rHsearchgrouprQrNAttributeErrorr rextendr!r"r$rTsetr5 intersectionr#r&r(sort)1r filter_given filter_dictr@rUrVrXrrr=r>r@r?r%rrrdrrhsql_fileryordered_resultsr/r0r1types_to_filter_outgenotype_only_typesgenotype_payloadgenotype_only_payloadsigenotype_only_typegenotype_only_filter_dictgenotype_only_payloadrgenotype_types sql_queriesgenotype_sql_querygeno_table_regexgenotype_only_sql_querygenotype_resp_rawgenotype_only_resp_raw ref_payload locus_id_refssamplesselected_samples loci_payloadlocusloci type_to_infers1 rYextract_assay_germliners L t774;Q;Q  ~~ ~   DHH B   88 J K [[ g  L M ~~**   4#;#;txx4K[K[ g    >> ARVWZaa_``cjjghhkrruvvy@@\] ]`g g tu u x  IZ^_ ` cg gaa  HHQK ! ! >> ARVWZaa~BIIKLLOVVGHHKRRbc cfm m W X X [ b b { | |  F F abbellijjmttM^bc dgk ko o  HHQK   >> ARVWZaa_``cjjoppszzUVWZaavw wzA A EF F IM M Y' ( +/ /YZ[^bbfghkooWX Y\` `bc dgk kef gjn nTU VY` `L]ab cfm m6tLMPTT"&*D\+.WM+.UK%*a  0127M3N2QU3U4""&". !=%J#5! D HHQK .x> ! !.|TB  .z4@   u   xx p  5QQUQZQZ4[1G]D/$}--#  4??+3  Q iJz?CRRTN^q $//499j._[*,?A[!5T4 ;Mz ZHoK  ! !" |,'$ #0#   dG, 88,T7;Kk"(C(6H+H566$D'2H$Xi%8>STO & /+(s8   , : > >?P Q : > >?P Q9==>NO    NN       >>T.."G XgikHl"m +-K ( + %"$ "9+:I?\j#l )6'$ #0"))) %+ d$45!# *33F)G E%%,7,<,<,>)%.:>-h7'61>B-l;'94=A-k:+8$=,($+%8 /$6%$?$6&$@#5#Bs#67!;; ,(%q&d,AB&--.CD1 E6,K8 88K%7>N%OPSQS%T"s',/IJ_/`'`ci'i$II.0BCIIK""#56*@ T%%h/66|7LM*Xs0f?/g 8h h ?g  -g=<g=h cg}t|djD]<}|jdj||d|d|d|ddus;|}>|fS)zc Retrieves the entities in form of and identifies main entity rz{} {} entity_titleis_main_entityT)rrwrrN)rrentity main_entitys rYrrs!z*//12!&& OOFE*$5f$=n$M N   V $%5 6$ > K ! #K //rnc  |dj}|d|g}|jrjg}g}t|jjdD]/}||vr|j |d||j |1|rt d|zg}|D]Q}t|djD]0}|j dj |d||d|d2Stdj|y ) zs Listing fileds in the model in form at . for specified list of entities rr}z1The following entity/entities cannot be found: %srz{}.{} {}rtitlerN) rwrrrrrrNrr) rrrpresent_entitiesentities_to_list_fieldsrrrrs rYrrsZ(--/$Z0=> }}"$ T]]0056 *F))'..uZ/@/HI!!&)  *  H:U V F)F8,1134 E MM""6NE6(+;E+B7+K   $))F rnrFr}rc |rtj}n t|d}tj|t |ddd||dd| }|j |D]} |j| |s|jyy)NrTrF) r doublequote escapecharlineterminator quotecharquotingskipinitialspacestrict fieldnames) rrrcsv DictWriterrR writeheaderwriterowclose) rrrrrrsr fields_output csv_writerrs rYrrs ]C0 c( J#E"#  rncb t|j|j|j|jg}|j r |r t d|jr-t|j|j|gr t d|jr>t|j|j|j|j|jgr t dn|j t dnttddd ztd zd ztd zd ztd zd ztd zd ztdzd ztdzd ztdzd ztdzd ztdzd ztdzd ztddzdzdztjd|jrt|j|j|gr t dnd}tt!ddzgdgdgdgdgdgd gd!g}||tdt!d"zdzgdgd#gd$gd%gd&gd'gd(gd)gd*gd+gd,gd-gd.gd/gd0gd1gd2gd3gd4gd5gd6gd7gd8g}||tjd|jQ|jj#d9Dcgc]}|j%}}gd:}|D]}||vst d;t'|j(\} } } } d<| d=vr,t|j|jgr t d>| d?} t+| | @j-}t/|j|j|j(dA|\}}}}t1|| dBdA\}}|j r%t | | ||||tjd|jr t3dC|}|jr t5|||| ||jD\}}nt5|||| ||jE\}}d<| d=vr-| j7dFr| j7dF|dG<| dH|dH<|jr=t9| |}|r t|yt;|dI5}t||Jdddyt=| |}t?||dK|dL|tAdKtBjDMyycc}w#1swYyxYw)Nz Retrieve the selected data or generate SQL to retrieve the data from an somatic variant assay in a dataset or cohort based on provided rules. z_The flag, --retrieve-meta-info cannot be used with arguments other than --assay-name, --output.rVzv--json-help cannot be passed with any of --assay-name, --sql, --additional-fields, --additional-fields-help, --output.Nz8--json-help cannot be passed without --retrieve-variant.rWrXrYrZalocation: "location" filters variants based on having an allele_id which has a corresponding annotation row which matches the supplied "chromosome" with CHROM and where the start position (POS) of the allele_id is between and including the supplied "starting_position" and "ending_position". If multiple values are provided in the list, the conditional search will be, "OR". String match is case sensitive.aVsymbol: "symbol" filters variants based on having an allele_id which has a corresponding annotation row which has a matching symbol (gene) name. If multiple values are provided, the conditional search will be, "OR". For example, ["BRCA2", "ASPM"], will search for variants which match either "BRCA2" or "ASPM". String match is case sensitive.agene: "gene" filters variants based on having an allele_id which has a corresponding annotation row which has a matching gene ID of the variant. If multiple values are provided, the conditional search will be, "OR". For example, ["ENSG00000302118", "ENSG00004000504"], will search for variants which match either "ENSG00000302118" or "ENSG00004000504". String match is case insensitive.afeature: "feature" filters variants based on having an allele_id which has a corresponding annotation row which has a matching feature ID. The most common Feature ID is a transcript_id. If multiple values are provided, the conditional search will be, "OR". For example, ["ENST00000302118", "ENST00004000504"], will search for variants which match either "ENST00000302118" or "ENST00004000504". String match is case insensitive.aThgvsc: "hgvsc" filters variants based on having an allele_id which has a corresponding annotation row which has a matching HGVSc. If multiple values are provided, the conditional search will be, "OR". For example, ["c.-49A>G", "c.-20T>G"], will search for alleles which match either "c.-49A>G" or "c.-20T>G". String match is case sensitive.aYhgvsp: "hgvsp" filters variants based on having an allele_id which has a corresponding annotation row which has a matching HGVSp. If multiple values are provided, the conditional search will be, "OR". For example, ["p.Gly2Asp", "p.Aps2Gly"], will search for variants which match either "p.Gly2Asp" or "p.Aps2Gly". String match is case sensitive.a!allele_id: "allele_id" filters variants based on allele_id match. If multiple values are provided, anymatch will be returned. For example, ["1_1000_A_T", "1_1010_C_T"], will search for allele_ids which match either "1_1000_A_T" or ""1_1010_C_T". String match is case sensitive/exact match.avariant_type: Type of allele. Accepted values are "SNP", "INS", "DEL", "DUP", "INV", "CNV", "CNV:TR", "BND", "DUP:TANDEM", "DEL:ME", "INS:ME", "MISSING", "MISSING:DEL", "UNSPECIFIED", "REF" or "OTHER". If multiple values are provided, the conditional search will be, "OR". For example, ["SNP", "INS"], will search for variants which match either "SNP" or ""INS". String match is case insensitive.aVsample_id: "sample_id" filters either a pair of tumor-normal samples based on having sample_id which has a corresponding sample row which has a matching sample_id. If a user has more than 500 IDs, it is recommended to either retrieve multiple times, or use a cohort id containing all desired individuals, providing the full set of sample_ids.a}assay_sample_id: "assay_sample_id" filters either a tumor or normal sample based on having an assay_sample_id which has a corresponding sample row which has a matching assay_sample_id. If a user has a list of more than 1,000 IDs, it is recommended to either retrieve multiple times, or use a cohort id containing all desired individuals, providing the full set of assay_sample_ids.z+JSON filter template for --retrieve-variantrao{ "location": [ { "chromosome": "1", "starting_position": "10000", "ending_position": "20000" }, { "chromosome": "X", "starting_position": "500", "ending_position": "1700" } ], "annotation": { "symbol": ["BRCA2"], "gene": ["ENST00000302118"], "feature": ["ENST00000302118.5"], "hgvsc": ["c.-49A>G"], "hgvsp": ["p.Gly2Asp"] }, "allele" : { "allele_id":["1_1000_A_T","2_1000_G_C"], "variant_type" : ["SNP", "INS"] }, "sample": { "sample_id": ["Sample1", "Sample2"], "assay_sample_id" : ["Sample1_tumt", "Sample1_nor"] } }rz@--additional-fields-help cannot be presented with other options.c |D]Y}dj|dd}t|}|tjddd|zzt |d| z }t |[y)Nz{: <22} {: <22} r r )rD)rNrrHrIr r)rrow fields_stringwidths rY print_fieldsz+extract_assay_somatic..print_fieldsisk!)C$=$6$=$=s2Aw$GM .E!RVVD$u2Dd3q6ejdjFk%llM-( )rnz8The following fields will always be returned by default:)NAMETITLE DESCRIPTION)assay_sample_idzAssay Sample IDzhA unique identifier for the tumor or normal sample. Populated from the sample columns of the VCF header.)rmz Allele IDz&An unique identification of the allele)CHROM Chromosomez1Chromosome of variant, verbatim from original VCF)POSPositionz8Starting position of variant, verbatim from original VCF)REFzReference Allelez5Reference allele of locus, verbatim from original VCF)rAllelezSequence of the allelezThe following fields may be added to the output by using option --additional-fields. If multiple fields are specified, use a comma to separate each entry. For example, "sample_id,tumor_normal")rPz Sample IDz-Unique ID of the pair of tumor-normal samples) tumor_normalz Tumor-Normalz9One of ["tumor", "normal"] to describe source sample type)rrzLComma separated list of associated IDs for the variant from the original VCF)QUALrz,Quality of locus, verbatim from original VCF)FILTERrz?Comma separated list of filters for locus from the original VCF)reference_sourcezReference SourcezSOne of ["GRCh37", "GRCh38"] or the allele_sample_id of the respective normal sample) variant_typez Variant Typez-The type of allele, with respect to reference) symbolic_typez Symbolic TypezJOne of ["precise", "imprecise"]. Non-symbolic alleles are always "precise")file_idzSource File IDz1DNAnexus platform file-id of original source file)INFOrz(INFO section, verbatim from original VCF)FORMATrz*FORMAT section, verbatim from original VCF)SYMBOLSymbolz/A list of gene name associated with the variant)GENOTYPErzLGENOTYPE section, as described by FORMAT section, verbatim from original VCF)normal_assay_sample_idzNormal Assay Sample IDz8Assay Sample ID of respective "normal" sample, if exists)normal_allele_idszNormal Allele IDsz5Allele ID(s) of respective "normal" sample, if exists)GenezGene IDz/A list of gene IDs, associated with the variant)Featurez Feature IDz2A list of feature IDs, associated with the variant)HGVScrz9A list of sequence variants in HGVS nomenclature, for DNA)HGVSprz=A list of sequence variants in HGVS nomenclature, for protein)CLIN_SIGzClinical Significancez5A list of allele specific clinical significance terms)ALTrzZAlternate allele(s) at locus, comma separated if more than one, verbatim from original VCF) alt_indexzALT allele_indexzOrder of the allele, as represented in the ALT field. If the allele is missing (i.e, "./0", "0/." or "./.") then the alt_index will be emptyr})rPrrrrrrrrrrrrrrrrrrrrrOne or more of the supplied fields using --additional-fields are invalid. Please run --additional-fields-help for a list of valid fieldsrLrKrnr]rr"rr)rorrErrpadditional_fieldsinclude_normal)rorrErrprrrrrrrr)rrrrrrsr)#rinclude_normal_samplerr}rbr rr6rrretrieve_variantrrNrr4additional_fields_helpr rrrZrSrrrBr&rr+rrmrr{rrRr QUOTE_NONE)rinvalid_combo_argsr fixed_fieldsradditional_fieldadditional_fields_inputaccepted_additional_fieldsrr@rUrVrXrrr=r>r@r?r%rrrdrrhrrys rYextract_assay_somaticr8s6 d88$:P:PRVR`R`bfbjbjkl #5 m  C$++?Q RS C  ~~ d.H.H$J`J`bfbjbjk l I  " " * O P ARVWZaavwwzAAuvvy@@abbellJK KNU U st t w~ ~ xy y |C C@AADKKkllovvuv vy@ @\] ]`g gIZ^_ `cg g m  m   HHQK "" .@A B R  ) $QRUYY Z<o`hkpJ LL  & $abbeii j!A!l!~!}!a!x"a!r"R!s!]!c!h"J"R"E!g!p!p!t!"N"T-!U . * + HHQK )TXTjTjTpTpqtTu"v@P#3#9#9#;"v"v&w", eE66de e 5QQUQZQZ4[1G]D/$}--#t7G7G6X2Y Q iJz?CRRTN^q $//499i_[*,?A[!5T4 ;My YHo 4*;=PRackl   .y$?  ! !#8 +($ '!4"9#99$ G[$9 +($ '!4#99 $ G[ d=1 1xx "&*hhy&9 #!%iGI  88,T7;Kk"(C(6H+H566$D'2H & /$Y/(t9 Q3#wx66s/T T%%T.c \1t|}t|tt}|j |j r$t ttjd|jr$t ttjd|jrt|j\}}}|&tdj|j|n|jd}||ddk7r%tdj|j|dd g}|dd 1|dd d k7st!1fd |Drtdj|d1t#d}t%j&|\} } | r|j(s |j*r tdt-| dddkrtdj| d| jdjd} | jdjd} d} nBd} d} d} t-| j.dkr$tdj| j0|j*r/t  j3ddditjd|j(rL j5|j(dstdj|j(| j0nY|j(sM j6jd't9| j6jddk(r tdtj:j<dk(rt>t@_!|jDr! tAjF|jD}n@|jLr4 tO|jL5}tAjP|}ddd|jVr|jDs|jLsd}i}nd}s |s td#|jXr&d$|vr td%|jZr td&tj:j<dk(rKt\t^d'd(d)d*d+<t\t^d'd(d)d,d+<t\t^d'd(d)d-d+<|r'tat^t.}|jc|/|j(r. j6dD]}|d0|j(k(s|d1}nn j6ddd1}tdtfd2}| jijd3}|j|}|d4k(rd'|vrtj}ntf}d'|vs|r:|d4k(rtj}|jVr d|d5d'd6<njm|d'd,d-d7d8to||t9}|jq}|d:jd;}|jZrg}|jZD]P}|jsd<Dcgc]#}|jus|ju%}}|jw|R|d:jd=} ty|ty| D!"chc]}!|!j{D]}"|"c}"}!z }#t9|#dk7r td>| D$cgc]*}$ty|$j{ty|zs)|$,}%}$|jw|%t}| rd ind|d?d | t@}&| j~}'|j(r |j(n| j3dd}(|&j|(||A|&j})| jd}*| j}+t|+|t},|jVr|,j|)|*}-n|,j|)|*}-d}.|jVr|-dBn|-dC}/|jXr|/rt|-dC\}0}.|0}/|/sig}/t|j|j|jV|/d| jd0|.Dy#t@jB$r$}d tI|z}t|Yd}~d}~wtJ$r}ttI|Yd}~d}~wwxYw#1swYxYw#t@jB$r$}d tI|z}t|Yd}~'d}~wtR$rQ}d!tI|vr%td"j|jTnttI|Yd}~d}~wtJ$r}ttI|Yd}~d}~wwxYwcc}wcc}"}!wcc}$w)Ez `dx extract_assay expression` Retrieve the selected data or generate SQL to retrieve the data from an expression assay in a dataset or cohort based on provided rules. ) parser_dictschema error_handlerrNz.Unable to resolve "{}" to a data object in {}.r r@zuUnable to resolve "{}" to a data object or folder name in {}. Please make sure the object is in the selected project.r.rLtypesrBrCc3&K|]}|v ywN).0x _record_typess rY z+extract_assay_expression..sA '(A] "A sz_{} Invalid path. The path must point to a record type of cohort or dataset and not a {} object.rEzCurrently "--assay-name" and "--list-assays" may not be used with a CohortBrowser record (Cohort Object) as input. To select a specific assay or to list assays, please use a Dataset object as input.rWversiong@zC{}: Version of the cohort is too old. Version must be at least 3.0.rrTFzD{}: Version of the dataset is too old. Version must be at least 3.0.molecular_expressionrrzAssay {} does not exist in {}, or the assay name provided cannot be recognized as a molecular expression assay. For valid assays accepted by the function, `extract_assay expression`, please use the --list-assays flag.z3No molecular expression assays found in the datasetr z9JSON provided for --retrieve-expression is malformatted. zNo such file or directoryz=JSON file {} provided to --retrieve-expression does not existzNo filter JSON is passed with --retrieve-expression or input JSON for --retrieve-expression does not contain valid filter information. expressionzvExpression filters are not compatible with --expression-matrix argument. Please remove "expression" from filters JSON.zE--additional-fields cannot be used with --expression-matrix argument.rir propertiesr_rsr`ending_position)rr) input_jsonrr+)z1.01.1generalized_assay_model_versionrfiltering_conditionsmax_item_limiti)rrq start_subkey end_subkey window_widthcheck_each_separately)rrroutput_fields_mappingdefaultr} additionalrorder_by)rr rrlimitrrr)rrrrbr)save_uncommon_delim_to_txtoutput_file_namecolnames)Krr/r1rvalidate_input_combinationr}rr=rr4rr>rSrrNrallr r.resolve_cohort_to_datasetrr6floatrrEassay_names_listis_assay_name_validassays_info_dictrrr ValueErrorrJSONDecodeErrorrrrRrQfilter_json_filerrOSErrorfilenamerbexpression_matrixrunicoder2r0validater3r4assay_info_dictr5are_list_items_within_ranger6parserrrrrwr7descriptor_file_dictassemble_assay_raw_filtersbuilddetail_describe vizserver_urlr8 get_raw_sqlget_datar9r;rr)2rrinput_validatorr@rTrUentity_describeEXPECTED_TYPESrCr] cohort_infoBASE_SQLCOHORT_FILTERS IS_COHORTuser_filters_jsonrjson_reading_errorrreturn_all_datainput_json_validatormolecular_assayASSAY_IDconditions_mappingr filter_schemainput_json_parservizserver_raw_filters_db_columns_listrrrrall_additional_colsdkincorrect_colsruser_additional_colsvizDATASET_DESCRIPTOR ASSAY_NAMEvizserver_payload record_idr\clientvizserver_responser output_datatransformed_responsers2 @rYextract_assay_expressionrNs t*K#9O ..0 ~~ 01   "" =>   yy.CDII.N+m  @GGIIw  ,// ;O mJ/ : : HOOIIw $_5%j1':  $W - 9SA ,:A >  qxx!$' /$/0&@@H $"2"2][+I67#=Y``#D) #y155i@H(__Y7;;IFNIH!NIW__%+Zaa    w''(>?JTJ   **4?? ? G3wOgOgOkOkmCPDLEIJLJ J K ")   $ 4+;+; <     d++, 1$(IIaL! 1, xx((1F1F _ U   , , I   ! ! W   " -Z8A,O     -Z8A,O     -Z8A,O    ,7x  %%1B%C  '778NO Ov&$//9*62  ++,BCAFvNED '.&=&=h&G&K&KLm&n#&**+JKM'%/ * *[MMM&&/ *e 3[M 88#' 0 1  ! < <,0,&&+ = -$ .335$ c)n  ** ,D(, 3E11779QWWYEEE  $ $U + ,, # #l  ./#+ >1QVVX >Q >Q >3   ~ ! # [ + c!&&(mcBS>T.TA    45 ./8N+dz* C!55??0H0HI_0`ab0c"":O# ''-I   CsGX .F xx#//0A9M#__-> JH&*XX5!3Ei3P  +)G y )* &h+ d    #' 008o## )a&  ' ( (  SV     1 1## )a&  ' ( ( !*c!f4SZZ  Q   SV   fF ?  sa9>c&c)c&8fff# "*f) f)9c b++ c7ccc#c&&f9d f$Ae00 f<ffcPt|\}}}d\}}|dk7r`tjj|}tjj |} t |||}|sdj||}||||fS#t $r}dt|vrd}n|Yd}~:d}~wwxYw)zc Resolves dx path into project, folder and name. Fails if non existing folder is provided. NNrDzfolder could not be foundFNz4The folder: {} could not be found in the project: {}) rrrSbasenamerrrrRrN) rSr@folderrerr_msg folder_exists folder_namerTrs rYresolve_validate_dx_pathrV5s).GVT'G] }gg&&v. ggoof-  /kRM LSSG FD' )) *c!f4 %  s B B% B  B%c eZdZy)VizserverErrorN)__name__ __module__ __qualname__rrnrYrXrXLsrnrXc|jdd}|jdd}|jd|d|dd}|dvrd }n-|d vrd }n%|d vrd }ndj|}t|t||g} dj||} || ig} || dd| d| dgiiid} t || } t}| dD]}|j|||t| k7r;t| j|}dj|d|}t|| |fS#t $r(}t djt|d}~wwxYw)Nglobal_primary_keyrrrrr-column_sql_type)integerbigintc |t|gzSr)rrbs rYz%validate_cohort_ids..YAs1vhJrn)rdoublec |t|gzSr)rrbs rYrdz%validate_cohort_ids..[sAuQxjLrn)rKc |t|gzSr)rRrbs rYrdz%validate_cohort_ids..]rernzInvalid input record. Cohort ID field in the input dataset or cohortbrowser record is of type, {type}. Support is currently only available for Cohort ID fields having one of the following types; string, integer and float)rsz{}${} pheno_filtersrin) conditionvalues)rrrzeException caught while validating cohort ids. Bad response from Vizserver.Original Error message: {}rz`The following supplied IDs do not match IDs in the main entity of dataset, {dataset_name}: {ids}r]) dataset_nameids) rrNrrr{rQrXrRradd difference) descriptorr@rVrn entity_name field_namegpk_typelambda_for_list_convrSid_listentity_field_namerrdryexcdiscovered_idsresult missing_idss rYvalidate_cohort_idsr|Os""#78BK!!"67@J +K8B:NyYZklH((6 ( (8 Z 6qxx@HxI!!)33G{J? 123K# %&*g>(  Gg.UN9%/6*-./W%'l--n= t{{LPQZL[cn{o!! ( ((%  *  &S  s- D.. E7#EEcgd}|j|} t|}|j|d|kryy#t$rYywxYw)z Validates that issuing user has required access level. Args: project: str: tasked project_id access_level: str: minimum requested level Retuns: boolean )VIEWUPLOAD CONTRIBUTE ADMINISTERFlevelT)indexr r)r@ access_level level_rankaccess_level_idxproject_describes rYhas_access_levelrsa@J!'' 5#G,(125EE  s ; AAc>t||sdj|Sy)z Validates that project has requested access. Args: project: str: tasked project_id access_level: str: minimum requested level Default at least UPLOAD Returns: Error message zBAt least {} permission is required to create a record in a projectN)rrN)r@rs rYvalidate_project_accessrs) G\ 2SZZ    3rnctj}tjjdd}d}|jr?t |j\}}}}|r t |t|}|r t ||jjd}t|\}}}} g} |jrGt|jd5} | D]"} | j| jd$ ddd|jr6|jjdD cgc]} | j} } t!|d| j#} t%|| || \}}|jd |jd } t/|j0d d|j0d d|jdi||}t5|}t7||||d|d|d|d|||jd }tj8j:j<di|}|j>rtA|jCy tE|jGdd|jHy#1swYxYwcc} w#t&$r&}t t)|t&f Yd}~Rd}~wt*$r&}t t)|t*f Yd}~d}~wt,$r}t t)|Yd}~d}~wwxYw#t,$r)}t dj3|d|Yd}~wd}~wwxYw#t,$r}t t)|Yd}~yd}~wwxYw)zL Create a cohort from dataset/cohort and specified list of samples. DX_CLI_WDrDNfromrrr}r]rrGrrr]rrrz{}: {}rE databasesrcombinedT)incl_properties incl_detailsr)%rO WORKSPACE_IDconfigrPATHrVrrrrZcohort_ids_filerrr cohort_idsrrrr|rrRrXrQr,rrNrpr-bindingsdxrecord new_dxrecordbriefrget_idrr verbose)r path_project path_folder path_namerSFROM from_projectrUrVrXrrrrEr list_of_idsruerrrraw_cohort_query_payloadrrbcohort_payload dx_records rY create_cohortrs!$$L++//+s3KI yy8P II9 5 k9g  W ),7  W  ==  V $D9UVZ9[6L- G  $&& , 1 1tzz$/0 1 1 (,(=(=c(BC"288:CCtIHWWYN,?P_aegn,o) )xx 488J#78H :#8   !5 6x @  !5 6w ? HHY #  $   $D*B CC) [ Y X +   N &&33EnEI zz i !  y))$T)RTXT`T` aq 1 1 D >S }== BS/@AA S" :t!4a899:.  SV   ss (J'JJAL'MJ L!K LK// L;LL M&M  M M9M44M9c4eZdZdZdZdfd ZdZdZxZS)ra9 A class to handle record objects of type Dataset. Inherits from DXRecord, but automatically populates default fields, details and properties. Attributes: All the same as DXRecord name - from record details description - from record details schema - from record details version - from record details descriptor - DXDatasetDescriptor object Functions get_descriptor - calls DXDatasetDescriptor(descriptor_dxfile) if descriptor is None get_dictionary - calls descriptor.get_dictionary r.cbtt| |||jdddh|j|j vsJd|j vsJt|j dr!t|j dd||_ ntd|j dzd|_ |j jd |_ |j jd |_|j jd |_|j jd |_y) NTrrW)default_fieldsrrqrb)moder@rIr descriptionrr)superr__init__r _record_typerrWr rdescriptor_dxfilerrqrrrrr)selfdxidr@ __class__s rYrzDXDataset.__init__s i'g6 T<2K L  DJJ...t||+++ T\\,/ 0%+DLL,FT[b%cD " 5 \8RR SLL$$V, <<++M:ll&&x0 ||'' 2 rnc~|j&t|j|j|_|jS)N)r)rqDXDatasetDescriptorrrrs rYrzDXDataset.get_descriptor,s3 ?? "1&&t{{DOrncn|j|j|jjSr)rqrrrs rYrzDXDataset.get_dictionary3s+ ?? "    !--//rnrP) rYrZr[__doc__rrrr __classcell__)rs@rYrr s"L 30rnrceZdZdZdZdZy)rz A class to represent a parsed descriptor of a Dataset record object. Attributes Representation of JSON object stored in descriptor file Functions get_dictionary - calls DXDatasetDictionary(descriptor) c tjjdk(xrtjjdk(}t |fddi|5}|rY|j }t |tk7r|jd}tj|tj}n%tj|tj}dddD]}t|||||jd|_y#1swY7xYw)Nris_gzipTzutf-8)object_pairs_hookr)rrrrrreadrsrRrrr collections OrderedDictrsetattrrr)rdxfilerM python3_5_xrjsonstrobjrqs rYrzDXDatasetDescriptor.__init__Ds&&,,1Qc6F6F6L6LPQ6Q v 6t 6v 6 N!&&(=C'%nnW5GjjKM &&*62d>6JJ >>#q($%5D %ABB#$=>EE#N3::")..-.5H0I#J#'~"6J - 4 4!$c!5!  $rncRgd}gd}dddddd}||zDcic]}|g}}|d gt|d z|d <d gt|d z|d <d gt|d z|d<|d D]} |d | } |d j| d |dj|| d|dj|dr| d |dk(r|rdndnd |dj| dr| dnd |dj| d|dj| d|djd| jvr| drdj| dnd |dj| drdnd |dj| drdnd |dj| d|dj| dr| dnd |dj| d|dj| d tj |} | Scc}w#t $r} | d } ~ wwxYw)!zH Returns DataDictionary pandas DataFrame for an entity. rrrsprimary_key_type) coding_nameconceptrrTis_multi_selectis_sparse_codinglinkoutlongitudinal_axis_typerrrunitsr_rdatedatetimerK)r_rfrrrKrrrrrrrsr primary_keygloballocalrrrrTz > ryesrrrrrN)rrrwrr DataFramer) rrrr]required_columns extra_colsdataset_datatype_dictcoldcolsr field_dictdframerxs rYrz(DXDatasetDictionary.create_entity_dframesJ  !" ! %5z$ABSbBB!&>*S1A-BBh,.4#fX6F2G+G'(!#s6(+;'< <nH%% 7E)%0J &M F!3 4 &M !6z&7I!J K $ % , ,=)"6*f].CC/G    - ' '-7 -F =)B  )  # #Jy$9 : - ' ' =(A B - ' '!Z__%66:m;T :m45  # $ + +#$56B  $ % , ,#$67R  )  # #Jy$9 : * + 2 26734  'N ! !*W"5 6 'N ! !*W"5 6K% 7N \\%(F cC\ I s H:H H&H!!H&ctj}|dD]}|d|d}dj|d|d|d}|d|f||<|dr?tj |dr'dj|d|d|d}|d|f||<x|ds~|d st j |dsd j|d j d d j|d}|d|f||<|S) z Returns map with "database$table$column", "unique_database$table$column", as keys and values are (entity, field) rr-z{}${}${} database_namerrrdatabase_unique_name database_idz{}__{}rr)rrrNdatabase_unique_name_regexmatchdatabase_id_regexrlower)rrrr field_valuedb_tb_col_pathunique_db_tb_col_pathunique_db_names rYrz5DXDatasetDictionary.get_join_path_to_entity_field_maps[ %0$;$;$=!H% TE *51)DF^U)(rnctj}|ddd}|ddd}||\|d<|d<||\|d<|d<|d |d <|S) zL Convert an item join_info to an edge. Returns ordereddict. rrrrrrrrr)rr)rjoin_info_joinsrr column_to column_froms rYrzDXDatasetDictionary.create_edges&&(#G,Q/5 %g.q1&9 6O 7 3_tN3 &k 2 % & $ %.~>^ rnctj}|jdD]`}|jd|dD]F}|jd|d|d}|s!||vs&|j|j|||||<Hb|S)z= Processes coding dictionary from descriptor rrr)rrrcreate_coding_name_dframe)rrqcblocksrrcoding_name_values rYrz*DXDatasetDictionary.load_coding_dictionary#s))+ &&z2 F#))*5f=hG $.$4$4Z$@$H$R%%!!%):')I151O1O"((&%9J2G-.   rnc Hi}|d|d|drd}fdt|d|dd|\}}}|j|||D cgc]} |d|d | c} |D cgc]8} |d|d r)| |d|d jvr|d|d | nd :c} |d nt|d|d j\} } |d|d r#t|d|d j\} } nd gt | z} | D cgc](} t |d|dj | d z*} } |j| | | | d|gt |dz|d< tj|}|Scc} wcc} wcc} w#t$r}|d }~wwxYw)zN Returns CodingDictionary pandas DataFrame for a coding_name. rris_hierarchicalrc3K|D]j}t|trLtt|j \}}|dz }|||f|||D] \}}}|||f_|dz }|||flyw)z~Serialize the node hierarchy by depth-first traversal. Yields: tuples of (code, parent_code) r:N) isinstancedictnextiterr) nodes parent_code displ_ordnodenext_parent_code child_nodes deep_node deep_parentunpack_hierarchys rYr(zGDXDatasetDictionary.create_coding_name_dframe..unpack_hierarchy;s " =D!$-8?@ST !$Y/0AB 3 "df!%i():;F-/\\-C-C#.(. >(B17.sort_dataframe_columnssL223CDJ/0:3I3I3K3R3R3TTK::an- -rnctj|jDcgc]}|c}d}||Scc}w)z(Join all blocks into a pandas DataFrame.F)r)rconcatrl)ord_dict_of_dfrrcdfrDs rY as_dataframez/DXDatasetDictionary.write..as_dataframes=~'<'<'>?!A?eLB)".>? ?@s >r)r)rr-r.rrN)rrto_csvrr) rrrrrcsv_optsrI data_dframe coding_dframe entity_dframerDs @rYrzDXDatasetDictionary.writes    .  @   &$$!OK K  / <8 <  ! !(&&!CM !M !3 @x @  ! !(&&(N9SM !M !3 @x @ "rnN)rrrr}) rYrZr[rrrrrrrrrrrrnrYrrYsCI 3jL\%)N" Xt!*VY*Arnr)T)z# )rVrrrRrrRreturnr5)r) __future__rrrrrrrrrrHrrOr subprocess functoolsrutils.printingr rr bindings.dxdataobject_functionsr r bindings.dxfilerutils.resolverrrrrrutils.file_handlerutils.describer exceptionsrrrrr"dx_extract_utils.filter_to_payloadrrdx_extract_utils.germline_utilsrrrr r!r"r#r$r%r&r'r(!dx_extract_utils.input_validationr))dx_extract_utils.input_validation_somaticr*'dx_extract_utils.somatic_filter_payloadr+&dx_extract_utils.cohort_filter_payloadr,r-bindings.apollo.datasetr.*bindings.apollo.cmd_line_options_validatorr/)bindings.apollo.json_validation_by_schemar0:bindings.apollo.schemas.input_arguments_validation_schemasr14bindings.apollo.schemas.assay_filtering_json_schemasr22bindings.apollo.schemas.assay_filtering_conditionsr3r4r52bindings.apollo.vizserver_filters_from_json_parserr6)bindings.apollo.vizserver_payload_builderr7bindings.apollo.vizclientr8$bindings.apollo.data_transformationsr9output_handlingr;r< help_messagesr=r>compiler r rZrirmrpr{rrrr r&rBrNrTrrrrR QUOTE_MINIMALrrrNrVrQrXr|rrrrrrrrnrYrns(`_  !A$qq)'N    EOK`-FEsgggUI1QGn'RZZ(@ABJJ3499x :A*|B~ 0 0f2'%RLd^v SGT 08 3x   DgROh *. Y ;)|&  Vr-0-0`))@rArArn