implemented the new payload-add-uniform-ids tool

Author: junjun-zhang

payload-add-uniform-ids/Dockerfile

@@ -1,9 +1,15 @@
-FROM ubuntu:20.04
+FROM python:3.7.5-slim-buster
 
 LABEL org.opencontainers.image.source https://github.com/icgc-argo/data-processing-utility-tools
 
 ENV PATH="/tools:${PATH}"
 
 COPY *.py /tools/
 
+RUN groupadd -g 1000 ubuntu && \
+    useradd -l -u 1000 -g ubuntu ubuntu && \
+    install -d -m 0755 -o ubuntu -g ubuntu /home/ubuntu
+
+USER ubuntu
+
 CMD ["/bin/bash"]

payload-add-uniform-ids/main.nf

@@ -44,8 +44,8 @@ params.publish_dir = ""  // set to empty string will disable publishDir
 
 
 // tool specific parmas go here, add / change as needed
-params.input_file = ""
-params.output_pattern = "*"  // output file name pattern
+params.payload_json = ""
+params.id_mapping_tsv = ""
 
 
 process payloadAddUniformIds {
@@ -56,10 +56,11 @@ process payloadAddUniformIds {
   memory "${params.mem} GB"
 
   input:  // input, make update as needed
-    path input_file
+    path payload_json
+    path id_mapping_tsv
 
   output:  // output, make update as needed
-    path "output_dir/${params.output_pattern}", emit: output_file
+    path "output_dir/*.json", emit: payload
 
   script:
     // add and initialize variables here as needed
@@ -68,7 +69,8 @@ process payloadAddUniformIds {
     mkdir -p output_dir
 
     main.py \
-      -i ${input_file} \
+      -p ${payload_json} \
+      -i ${id_mapping_tsv} \
       -o output_dir
 
     """
@@ -79,6 +81,7 @@ process payloadAddUniformIds {
 // using this command: nextflow run <git_acc>/<repo>/<pkg_name>/<main_script>.nf -r <pkg_name>.v<pkg_version> --params-file xxx
 workflow {
   payloadAddUniformIds(
-    file(params.input_file)
+    file(params.payload_json),
+    file(params.id_mapping_tsv)
   )
 }

payload-add-uniform-ids/main.py

@@ -24,32 +24,97 @@
 import os
 import sys
 import argparse
-import subprocess
+import csv
+import json
+
+
+def get_id_mapping(id_mapping_tsv):
+    id_mapping = dict()
+    with open(id_mapping_tsv, 'r') as f:
+        for row in csv.DictReader(f, delimiter='\t'):
+            type = row['type']
+            submitter_id = row['submitter_id']
+            uniform_id = row['uniform_id']
+            if type in id_mapping:
+                sys.exit(f"Values in 'type' field duplicated. Offending value: {type}, in file: {args.id_mapping_tsv}")
+            else:
+                id_mapping[type] = dict()
+
+            if submitter_id in id_mapping[type]:
+                sys.exit(f"Values in 'submitter_id' field duplicated. Offending value: {submitter_id}, for type: {type}, in file: {args.id_mapping_tsv}" )
+            else:
+                id_mapping[type][submitter_id] = uniform_id
+
+    if 'donor' not in id_mapping or 'specimen' not in id_mapping or 'sample' not in id_mapping:
+        sys.exit(f"Provided id_mapping_tsv file '{args.id_mapping_tsv}' is required to have ID mappings for 'donor', 'specimen' and 'sample'")
+
+    return id_mapping
+
+
+def add_uniform_ids(payload, id_mapping):
+    samples = payload.pop('samples', [])
+    if not samples:
+        sys.exit("Error: no 'samples' found in the input payload JSON")
+
+    updated_samples = []
+    for sample in samples:
+        if id_mapping['sample'].get(sample['submitterSampleId']):
+            sample['sampleId'] = id_mapping['sample'][sample['submitterSampleId']]
+        else:
+            sys.exit(f"Provided id_mapping_tsv misses mapping for submitter sample ID: {sample['submitterSampleId']}")
+
+        if id_mapping['specimen'].get(sample['specimen']['submitterSpecimenId']):
+            sample['specimenId'] = id_mapping['specimen'][sample['specimen']['submitterSpecimenId']]
+            sample['specimen']['specimenId'] = sample["specimenId"]
+        else:
+            sys.exit(f"Provided id_mapping_tsv misses mapping for submitter specimen ID: {sample['specimen']['submitterSpecimenId']}")
+
+        if id_mapping['donor'].get(sample['donor']['submitterDonorId']):
+            sample['donor']['donorId'] = id_mapping['donor'][sample['donor']['submitterDonorId']]
+            sample['specimen']['donorId'] = sample['donor']['donorId']
+        else:
+            sys.exit(f"Provided id_mapping_tsv misses mapping for submitter donor ID: {sample['donor']['submitterDonorId']}")
+
+        updated_samples.append(sample)
+
+    payload['samples'] = updated_samples
 
 
 def main():
     """
-    Python implementation of tool: payload-add-uniform-ids
-
-    This is auto-generated Python code, please update as needed!
+    Add uniform IDs for donor/specimen/sample to the original input payload JSON
     """
 
     parser = argparse.ArgumentParser(description='Tool: payload-add-uniform-ids')
-    parser.add_argument('-i', '--input-file', dest='input_file', type=str,
-                        help='Input file', required=True)
-    parser.add_argument('-o', '--output-dir', dest='output_dir', type=str,
+    parser.add_argument('-p', '--payload-json', type=str,
+                        help='Input payload JSON', required=True)
+    parser.add_argument('-i', '--id-mapping-tsv', type=str,
+                        help='TSV file containing mapping between submitter IDs and uniform IDs', required=True)
+    parser.add_argument('-o', '--output-dir', type=str,
                         help='Output directory', required=True)
     args = parser.parse_args()
 
-    if not os.path.isfile(args.input_file):
-        sys.exit('Error: specified input file %s does not exist or is not accessible!' % args.input_file)
+    if not os.path.isfile(args.payload_json):
+        sys.exit('Error: specified input payload JSON %s does not exist or is not accessible!' % args.payload_json)
+
+    if not os.path.isfile(args.id_mapping_tsv):
+        sys.exit('Error: specified ID mapping TSV %s does not exist or is not accessible!' % args.id_mapping_tsv)
 
     if not os.path.isdir(args.output_dir):
         sys.exit('Error: specified output dir %s does not exist or is not accessible!' % args.output_dir)
 
-    subprocess.run(f"cp {args.input_file} {args.output_dir}/", shell=True, check=True)
+    with open(args.payload_json, 'r') as p:
+        payload = json.loads(p.read())
+
+    id_mapping = get_id_mapping(args.id_mapping_tsv)
+
+    add_uniform_ids(payload, id_mapping)
+
+    output_payload_file = "%s.uniform_id_added.json" % os.path.splitext(os.path.basename(args.payload_json))[0]
+
+    with open(os.path.join(args.output_dir, output_payload_file), 'w') as o:
+        o.write(json.dumps(payload, indent=2))
 
 
 if __name__ == "__main__":
     main()
-

payload-add-uniform-ids/tests/checker.nf

@@ -43,7 +43,8 @@ params.container_version = ""
 params.container = ""
 
 // tool specific parmas go here, add / change as needed
-params.input_file = ""
+params.payload_json = ""
+params.id_mapping_tsv = ""
 params.expected_output = ""
 
 include { payloadAddUniformIds } from '../main'
@@ -61,37 +62,35 @@ process file_smart_diff {
 
   script:
     """
-    # Note: this is only for demo purpose, please write your own 'diff' according to your own needs.
-    # remove date field before comparison eg, <div id="header_filename">Tue 19 Jan 2021<br/>test_rg_3.bam</div>
-    # sed -e 's#"header_filename">.*<br/>test_rg_3.bam#"header_filename"><br/>test_rg_3.bam</div>#'
-
-    diff <( cat ${output_file} | sed -e 's#"header_filename">.*<br/>#"header_filename"><br/>#' ) \
-         <( ([[ '${expected_file}' == *.gz ]] && gunzip -c ${expected_file} || cat ${expected_file}) | sed -e 's#"header_filename">.*<br/>#"header_filename"><br/>#' ) \
+    diff ${output_file} ${expected_file} \
     && ( echo "Test PASSED" && exit 0 ) || ( echo "Test FAILED, output file mismatch." && exit 1 )
     """
 }
 
 
 workflow checker {
   take:
-    input_file
+    payload_json
+    id_mapping_tsv
     expected_output
 
   main:
     payloadAddUniformIds(
-      input_file
+      payload_json,
+      id_mapping_tsv
     )
 
     file_smart_diff(
-      payloadAddUniformIds.out.output_file,
+      payloadAddUniformIds.out.payload,
       expected_output
     )
 }
 
 
 workflow {
   checker(
-    file(params.input_file),
+    file(params.payload_json),
+    file(params.id_mapping_tsv),
     file(params.expected_output)
   )
 }

payload-add-uniform-ids/tests/expected/expected.test_rg_3.bam

@@ -0,0 +1 @@
+../../../tests/data/cab531fd-dc75-462f-8a73-fb7fcbf73185.id_mapping.tsv

payload-add-uniform-ids/tests/expected/expected.test_rg_3_fastqc.out.gz

@@ -0,0 +1 @@
+../../../tests/data/cab531fd-dc75-462f-8a73-fb7fcbf73185.sequencing_experiment.payload.json

payload-add-uniform-ids/tests/input/cab531fd-dc75-462f-8a73-fb7fcbf73185.id_mapping.tsv

@@ -0,0 +1 @@
+../../../tests/data/dc51bfb0-c073-4abd-a0da-fb67aa73d58b.id_mapping.tsv

payload-add-uniform-ids/tests/input/cab531fd-dc75-462f-8a73-fb7fcbf73185.sequencing_experiment.payload.json

@@ -0,0 +1 @@
+../../../tests/data/dc51bfb0-c073-4abd-a0da-fb67aa73d58b.sequencing_experiment.payload.json