added support for extra_info_tsv to provide mapping for uniform IDs

Author: junjun-zhang

payload-gen-seq-experiment/main.nf

@@ -45,10 +45,10 @@ params.publish_dir = ""  // set to empty string will disable publishDir
 
 
 // tool specific parmas go here, add / change as needed
-params.metadata_json = "NO_FILE1"
-params.experiment_info_tsv = "NO_FILE2"
-params.read_group_info_tsv = "NO_FILE3"
-params.file_info_tsv = "NO_FILE4"
+params.experiment_info_tsv = "NO_FILE1"
+params.read_group_info_tsv = "NO_FILE2"
+params.file_info_tsv = "NO_FILE3"
+params.extra_info_tsv = "NO_FILE4"
 
 
 process payloadGenSeqExperiment {
@@ -59,26 +59,26 @@ process payloadGenSeqExperiment {
   memory "${params.mem} GB"
 
   input:
-    path metadata_json
     path experiment_info_tsv
     path read_group_info_tsv
     path file_info_tsv
+    path extra_info_tsv
 
   output:
     path "*.sequencing_experiment.payload.json", emit: payload
 
   script:
-    args_metadata_json = !metadata_json.name.startsWith("NO_FILE") ? "-m ${metadata_json}" : ""
     args_experiment_info_tsv = !experiment_info_tsv.name.startsWith("NO_FILE") ? "-x ${experiment_info_tsv}" : ""
     args_read_group_info_tsv = !read_group_info_tsv.name.startsWith("NO_FILE") ? "-r ${read_group_info_tsv}" : ""
     args_file_info_tsv = !file_info_tsv.name.startsWith("NO_FILE") ? "-f ${file_info_tsv}" : ""
+    args_extra_info_tsv = !extra_info_tsv.name.startsWith("NO_FILE") ? "-e ${extra_info_tsv}" : ""
 
     """
     main.py \
-         ${args_metadata_json} \
          ${args_experiment_info_tsv} \
          ${args_read_group_info_tsv} \
-         ${args_file_info_tsv}
+         ${args_file_info_tsv} \
+         ${args_extra_info_tsv}
     """
 }
 
@@ -87,9 +87,9 @@ process payloadGenSeqExperiment {
 // using this command: nextflow run <git_acc>/<repo>/<pkg_name>/<main_script>.nf -r <pkg_name>.v<pkg_version> --params-file xxx
 workflow {
   payloadGenSeqExperiment(
-    file(params.metadata_json),
     file(params.experiment_info_tsv),
     file(params.read_group_info_tsv),
-    file(params.file_info_tsv)
+    file(params.file_info_tsv),
+    file(params.extra_info_tsv)
   )
 }

payload-gen-seq-experiment/main.py

@@ -159,7 +159,7 @@ def validate_args(args):
         ))
 
 
-def main(metadata):
+def main(metadata, extra_info=dict()):
     empty_str_to_null(metadata)
 
     payload = {
@@ -198,6 +198,24 @@ def main(metadata):
         }
     }
 
+    if extra_info:
+        if extra_info['sample'].get(sample['submitterSampleId']):
+            sample['sampleId'] = extra_info['sample'][sample['submitterSampleId']]
+        else:
+            sys.exit(f"Provided extra_info_tsv misses mapping for submitter sample ID: {sample['submitterSampleId']}")
+
+        if extra_info['specimen'].get(sample['specimen']['submitterSpecimenId']):
+            sample['specimenId'] = extra_info['specimen'][sample['specimen']['submitterSpecimenId']]
+            sample['specimen']['specimenId'] = sample["specimenId"]
+        else:
+            sys.exit(f"Provided extra_info_tsv misses mapping for submitter specimen ID: {sample['specimen']['submitterSpecimenId']}")
+
+        if extra_info['donor'].get(sample['donor']['submitterDonorId']):
+            sample['donor']['donorId'] = extra_info['donor'][sample['donor']['submitterDonorId']]
+            sample['specimen']['donorId'] = sample['donor']['donorId']
+        else:
+            sys.exit(f"Provided extra_info_tsv misses mapping for submitter donor ID: {sample['donor']['submitterDonorId']}")
+
     payload['samples'].append(sample)
 
     # get file of the payload
@@ -235,6 +253,8 @@ def main(metadata):
                         help="tsv file containing read_group information submitted from user")
     parser.add_argument("-f", "--file-info-tsv",
                         help="tsv file containing file information submitted from user")
+    parser.add_argument("-e", "--extra-info-tsv",
+                        help="tsv file containing file information submitted from user")
     args = parser.parse_args()
 
     validate_args(args)
@@ -258,4 +278,24 @@ def main(metadata):
         # all TSV are well-formed, let's load them
         metadata = load_all_tsvs(args.experiment_info_tsv, args.read_group_info_tsv, args.file_info_tsv)
 
-    main(metadata)
+    extra_info = dict()
+    if args.extra_info_tsv:
+        with open(args.extra_info_tsv, 'r') as f:
+            for row in csv.DictReader(f, delimiter='\t'):
+                type = row['type']
+                submitter_id = row['submitter_id']
+                uniform_id = row['uniform_id']
+                if type in extra_info:
+                    sys.exit(f"Values in 'type' field duplicated. Offending value: {type}, in file: {args.extra_info_tsv}")
+                else:
+                    extra_info[type] = dict()
+
+                if submitter_id in extra_info[type]:
+                    sys.exit(f"Values in 'submitter_id' field duplicated. Offending value: {submitter_id}, for type: {type}, in file: {args.extra_info_tsv}" )
+                else:
+                    extra_info[type][submitter_id] = uniform_id
+
+        if 'donor' not in extra_info or 'specimen' not in extra_info or 'sample' not in extra_info:
+            sys.exit(f"Provided extra_info_tsv file '{args.extra_info_tsv}' is required to have ID mappings for 'donor', 'specimen' and 'sample'")
+
+    main(metadata, extra_info)

payload-gen-seq-experiment/tests/checker.nf

@@ -29,7 +29,7 @@
 /* this block is auto-generated based on info from pkg.json where   */
 /* changes can be made if needed, do NOT modify this block manually */
 nextflow.enable.dsl = 2
-version = '0.3.0'  // package version
+version = '0.4.0'
 
 container = [
     'ghcr.io': 'ghcr.io/icgc-argo/data-processing-utility-tools.payload-gen-seq-experiment'
@@ -43,10 +43,10 @@ params.container_version = ""
 params.container = ""
 
 // tool specific parmas go here, add / change as needed
-params.metadata_json = "NO_FILE1"
-params.experiment_info_tsv = "NO_FILE2"
-params.read_group_info_tsv = "NO_FILE3"
-params.file_info_tsv = "NO_FILE4"
+params.experiment_info_tsv = "NO_FILE1"
+params.read_group_info_tsv = "NO_FILE2"
+params.file_info_tsv = "NO_FILE3"
+params.extra_info_tsv = "NO_FILE4"
 params.expected_output = ""
 
 include { payloadGenSeqExperiment } from '../main'
@@ -77,18 +77,18 @@ process file_smart_diff {
 
 workflow checker {
   take:
-    metadata_json
     experiment_info_tsv
     read_group_info_tsv
     file_info_tsv
+    extra_info_tsv
     expected_output
 
   main:
     payloadGenSeqExperiment(
-      metadata_json,
       experiment_info_tsv,
       read_group_info_tsv,
-      file_info_tsv
+      file_info_tsv,
+      extra_info_tsv
     )
 
     file_smart_diff(
@@ -100,10 +100,10 @@ workflow checker {
 
 workflow {
   checker(
-    file(params.metadata_json),
     file(params.experiment_info_tsv),
     file(params.read_group_info_tsv),
     file(params.file_info_tsv),
+    file(params.extra_info_tsv),
     file(params.expected_output)
   )
 }

payload-gen-seq-experiment/tests/input/extra_info.v2.tsv

@@ -0,0 +1 @@
+../../../tests/data/extra_info.v2.tsv

payload-gen-seq-experiment/tests/test-job-bam.nf.json

@@ -2,6 +2,7 @@
   "experiment_info_tsv": "input/experiment.v2.tsv",
   "read_group_info_tsv": "input/read_group.v2.tsv",
   "file_info_tsv": "input/file.v2.tsv",
+  "extra_info_tsv": "input/extra_info.v2.tsv",
   "expected_output": "input/98690064-627e-4a2b-a0c2-60fc9ca8433c.sequencing_experiment.payload.json",
   "publish_dir": "outdir"
 }

payload-gen-seq-experiment/tests/test-job-bam.tsv.nf.json

@@ -62,12 +62,17 @@
         "submitterSpecimenId": "HCC1143_BAM_INPUT",
         "tumourNormalDesignation": "Normal",
         "specimenTissueSource": "Blood derived",
-        "specimenType": "Cell line - derived from normal"
+        "specimenType": "Cell line - derived from normal",
+        "specimenId": "SP2222222222",
+        "donorId": "DO1111111111"
       },
       "donor": {
         "submitterDonorId": "HCC1143",
-        "gender": "Female"
-      }
+        "gender": "Female",
+        "donorId": "DO1111111111"
+      },
+      "sampleId": "SA3333333333",
+      "specimenId": "SP2222222222"
     }
   ],
   "files": [

payload-gen-seq-experiment/tests/test-job-fq.nf.json

@@ -0,0 +1,4 @@
+type	submitter_id	uniform_id
+donor	HCC1143	DO1111111111
+specimen	HCC1143_BAM_INPUT	SP2222222222
+sample	HCC1143_BAM_INPUT	SA3333333333