Merge pull request #102 from icgc-argo/payload-gen-seq-experiment.0.2.0.0

update payload-gen-seq-experiment

Author: junjun-zhang

tests/data/experiment-fq.v2.tsv

@@ -0,0 +1,2 @@
+type	program_id	submitter_sequencing_experiment_id	submitter_donor_id	gender	submitter_specimen_id	tumour_normal_designation	specimen_type	specimen_tissue_source	submitter_sample_id	sample_type	submitter_matched_normal_sample_id	sequencing_center	platform	platform_model	experimental_strategy	sequencing_date	read_group_count
+sequencing_experiment	TEST-PRO	TEST_EXP	HCC1143	Female	HCC1143_FASTQ_INPUT	Tumour	Cell line - derived from tumour	Blood derived	HCC1143_FASTQ_INPUT	DNA	HCC1143_BAM_INPUT	EXT	ILLUMINA	HiSeq 2000	WGS	2014-12-12	3

tests/data/experiment.v2.tsv

@@ -0,0 +1,2 @@
+type	program_id	submitter_sequencing_experiment_id	submitter_donor_id	gender	submitter_specimen_id	tumour_normal_designation	specimen_type	specimen_tissue_source	submitter_sample_id	sample_type	submitter_matched_normal_sample_id	sequencing_center	platform	platform_model	experimental_strategy	sequencing_date	read_group_count
+sequencing_experiment	TEST-PRO	TEST_EXP	HCC1143	Female	HCC1143_BAM_INPUT	Normal	Cell line - derived from normal	Blood derived	HCC1143_BAM_INPUT	DNA		EXT	ILLUMINA	HiSeq 2000	WGS	2014-12-12	3

tests/data/file-fq.v2.tsv

@@ -0,0 +1,7 @@
+type	name	format	size	md5sum	path
+file	C0HVY.2_r1.fq	FASTQ	5381	6584ebb05edfbd6f59be6307556bd871	C0HVY.2_r1.fq
+file	C0HVY.2_r2.fq	FASTQ	5381	d649696f346eb75b95b68e8ecd45f44f	C0HVY.2_r2.fq
+file	D0RE2.1_r1.fq	FASTQ	6148	058510bbdab2dccc14a2d7402a21248d	D0RE2.1_r1.fq
+file	D0RE2.1_r2.fq	FASTQ	6148	5fb35647198666b6a06e8539074282fd	D0RE2.1_r2.fq
+file	D0RH0.2_r1.fq	FASTQ	4784	eb109b8774df03084024bc09a86721be	D0RH0.2_r1.fq
+file	D0RH0.2_r2.fq	FASTQ	4784	ed209fda15a5928087104253a6bfb42d	D0RH0.2_r2.fq

tests/data/file.v2.tsv

@@ -0,0 +1,2 @@
+type	name	format	size	md5sum	path
+file	test_rg_3.bam	BAM	14911	178f97f7b1ca8bfc28fd5586bdd56799	test_rg_3.bam

tests/data/read_group-fq.v2.tsv

@@ -0,0 +1,4 @@
+type	submitter_read_group_id	read_group_id_in_bam	submitter_sequencing_experiment_id	platform_unit	is_paired_end	file_r1	file_r2	read_length_r1	read_length_r2	insert_size	sample_barcode	library_name
+read_group	C0HVY.2		TEST_EXP	74_8a	true	C0HVY.2_r1.fq	C0HVY.2_r2.fq	150	150	298		Pond-147580
+read_group	D0RE2.1		TEST_EXP	74_8b	true	D0RE2.1_r1.fq	D0RE2.1_r2.fq	150	150	298		Pond-147580
+read_group	D0RH0.2		TEST_EXP	74_8c	true	D0RH0.2_r1.fq	D0RH0.2_r2.fq	150	150	298		Pond-147580

tests/data/read_group.v2.tsv

@@ -0,0 +1,4 @@
+type	submitter_read_group_id	read_group_id_in_bam	submitter_sequencing_experiment_id	platform_unit	is_paired_end	file_r1	file_r2	read_length_r1	read_length_r2	insert_size	sample_barcode	library_name
+read_group	C0HVY.2		TEST_EXP	74_8a	true	test_rg_3.bam	test_rg_3.bam	150	150	298		Pond-147580
+read_group	D0RE2.1		TEST_EXP	74_8b	true	test_rg_3.bam	test_rg_3.bam	150	150	298		Pond-147580
+read_group	D0RH0.2		TEST_EXP	74_8c	true	test_rg_3.bam	test_rg_3.bam	150	150	298		Pond-147580

tools/payload-gen-seq-experiment/payload-gen-seq-experiment.nf

@@ -22,34 +22,38 @@
  */
 
 nextflow.enable.dsl=2
-version = '0.1.3.0'
+version = '0.2.0.0'
+
+params.metadata_json = "NO_FILE1"
+params.experiment_info_tsv = "NO_FILE2"
+params.read_group_info_tsv = "NO_FILE3"
+params.file_info_tsv = "NO_FILE4"
 
-params.user_submit_metadata = ""
-params.wf_name = ""
-params.wf_short_name = ""
-params.wf_version = ""
 params.container_version = ''
 
 process payloadGenSeqExperiment {
   container "quay.io/icgc-argo/payload-gen-seq-experiment:payload-gen-seq-experiment.${params.container_version ?: version}"
 
   input:
-    path user_submit_metadata
-    val wf_name
-    val wf_short_name
-    val wf_version
-    val seq_valid
+    path metadata_json
+    path experiment_info_tsv
+    path read_group_info_tsv
+    path file_info_tsv
 
   output:
     path "*.sequencing_experiment.payload.json", emit: payload
 
   script:
-    args_wf_short_name = wf_short_name.length() > 0 ? "-c ${wf_short_name}" : ""
+    args_metadata_json = !metadata_json.name.startsWith("NO_FILE") ? "-m ${metadata_json}" : ""
+    args_experiment_info_tsv = !experiment_info_tsv.name.startsWith("NO_FILE") ? "-x ${experiment_info_tsv}" : ""
+    args_read_group_info_tsv = !read_group_info_tsv.name.startsWith("NO_FILE") ? "-r ${read_group_info_tsv}" : ""
+    args_file_info_tsv = !file_info_tsv.name.startsWith("NO_FILE") ? "-f ${file_info_tsv}" : ""
+
     """
     payload-gen-seq-experiment.py \
-         -m ${user_submit_metadata} \
-         -w ${wf_name} \
-         -r ${workflow.runName} \
-         -v ${wf_version} ${args_wf_short_name}
+         ${args_metadata_json} \
+         ${args_experiment_info_tsv} \
+         ${args_read_group_info_tsv} \
+         ${args_file_info_tsv}
     """
 }

tools/payload-gen-seq-experiment/payload-gen-seq-experiment.py

@@ -22,11 +22,31 @@
  """
 
 
+import sys
 import uuid
 import json
+import csv
+import textwrap
 from argparse import ArgumentParser
 
 
+TSV_FIELDS = {
+    'experiment': [
+        'type', 'program_id', 'submitter_sequencing_experiment_id', 'submitter_donor_id', 'gender',
+        'submitter_specimen_id', 'tumour_normal_designation', 'specimen_type', 'specimen_tissue_source', 'submitter_sample_id',
+        'sample_type', 'submitter_matched_normal_sample_id', 'sequencing_center', 'platform', 'platform_model',
+        'experimental_strategy', 'sequencing_date', 'read_group_count'
+    ],
+    'read_group': [
+        'type', 'submitter_read_group_id', 'read_group_id_in_bam', 'submitter_sequencing_experiment_id', 'platform_unit',
+        'is_paired_end', 'file_r1', 'file_r2', 'read_length_r1', 'read_length_r2', 'insert_size', 'sample_barcode', 'library_name'
+    ],
+    'file': [
+        'type', 'name', 'size', 'md5sum', 'path', 'format'
+    ]
+}
+
+
 def empty_str_to_null(metadata):
     for k in metadata:
         if k in ['read_groups', 'files']:
@@ -36,10 +56,110 @@ def empty_str_to_null(metadata):
             metadata[k] = None
 
 
-def main(args):
-    with open(args.user_submit_metadata, 'r') as f:
-        metadata = json.load(f)
+def tsv_confomity_check(ftype, tsv):
+    expected_fields = TSV_FIELDS[ftype]
+
+    header_processed = False
+    with open(tsv, 'r') as t:
+        uniq_row = {}
+        for l in t:
+            l = l.rstrip('\n').rstrip('\r')  # remove trailing newline, remove windows `\r` (just in case)
+            if not header_processed:  # it's header
+                fields = l.split('\t')
+                if len(fields) != len(set(fields)):
+                    sys.exit("Error found: Field duplicated in input TSV: %s, offending header: %s\n" % (tsv, l))
+
+                missed_fields = set(expected_fields) - set(fields)
+                if missed_fields:  # missing fields
+                    sys.exit("Error found: Field missing in input TSV: %s, offending header: %s. Missed field(s): %s\n" % \
+                        (tsv, l, ', '.join(missed_fields)))
+
+                unexpected_fields = set(fields) - set(expected_fields)
+                if unexpected_fields:  # unexpected fields
+                    sys.exit("Error found: Unexpected field in input TSV: %s, offending header: %s. Unexpected field(s): %s\n" % \
+                        (tsv, l, ', '.join(unexpected_fields)))
+
+                header_processed = True
+
+            else:  # it's data row
+                # at this point we only check whether number of values matches number of expected fields and uniqueness check,
+                # later steps will perform more sophisticated content check
+                values = l.split('\t')
+                if len(expected_fields) != len(values):
+                    sys.exit("Error found: number of fields: %s does not match expected: %s, offending data row: %s\n" % \
+                        (len(values), len(expected_fields), l))
+
+                if l in uniq_row:
+                    sys.exit("Error found: data row repeated in file: %s, offending data row: %s\n" % (tsv, l))
+                else:
+                    uniq_row[l] = True
+
+
+def load_all_tsvs(exp_tsv, rg_tsv, file_tsv):
+    metadata_dict = {}
+    with open(exp_tsv, 'r') as f:
+        rows = list(csv.DictReader(f, delimiter='\t'))
+        if len(rows) != 1:
+            sys.exit("Error found: experiment TSV expects exactly one data row, offending file: %s has %s row(s)\n" % \
+                (exp_tsv, len(rows)))
+        rows[0]['read_group_count'] = int(rows[0]['read_group_count'])
+        metadata_dict.update(rows[0])
+
+    with open(rg_tsv, 'r') as f:
+        metadata_dict['read_groups'] = []
+        for rg in csv.DictReader(f, delimiter='\t'):
+            if rg['is_paired_end'].lower() == 'true':
+                rg['is_paired_end'] = True
+            elif rg['is_paired_end'].lower() == 'false':
+                rg['is_paired_end'] = False
+            else:
+                rg['is_paired_end'] = None
+
+            for field in ('read_length_r1', 'read_length_r2', 'insert_size'):
+                if rg[field]:
+                    rg[field] = int(rg[field])
+                else:
+                    rg[field] = None
+
+            metadata_dict['read_groups'].append(rg)
 
+        if len(metadata_dict['read_groups']) == 0:
+            sys.exit("Error found: read group TSV does not contain any read group information\n")
+
+    with open(file_tsv, 'r') as f:
+        metadata_dict['files'] = []
+        for f in csv.DictReader(f, delimiter='\t'):
+            if f['size']:
+                f['size'] = int(f['size'])
+            else:
+                f['size'] = None
+
+            metadata_dict['files'].append(f)
+
+        if len(metadata_dict['files']) == 0:
+            sys.exit("Error found: file TSV does not contain any file information\n")
+
+    return metadata_dict
+
+
+def validate_args(args):
+    if args.metadata_json and \
+            not (args.experiment_info_tsv or args.read_group_info_tsv or args.file_info_tsv):
+        return True
+    elif not args.metadata_json and \
+            (args.experiment_info_tsv and args.read_group_info_tsv and args.file_info_tsv):
+        return True
+    else:
+        sys.exit(textwrap.dedent(
+            """
+            Usage:
+                When '-m' is provided, no other arguments can be used
+                When '-m' is not provided, please provide all of these arguments: -x, -r and -f
+            """
+        ))
+
+
+def main(metadata):
     empty_str_to_null(metadata)
 
     payload = {
@@ -89,7 +209,10 @@ def main(args):
                 'fileMd5sum': input_file.get('md5sum'),
                 'fileType': input_file.get('format'),
                 'fileAccess': 'controlled',
-                'dataType': 'submitted_reads'
+                'dataType': 'Submitted Reads',
+                'info': {
+                    'data_category': 'Sequencing Reads'
+                }
             }
         )
 
@@ -104,12 +227,35 @@ def main(args):
 
 if __name__ == "__main__":
     parser = ArgumentParser()
-    parser.add_argument("-m", "--user-submit-metadata", dest="user_submit_metadata", required=True,
+    parser.add_argument("-m", "--metadata-json",
                         help="json file containing experiment, read_group and file information submitted from user")
-    parser.add_argument("-w", "--wf-name", dest="wf_name", type=str, help="workflow full name", required=True)
-    parser.add_argument("-c", "--wf-short-name", dest="wf_short_name", type=str, help="workflow short name")
-    parser.add_argument("-v", "--wf-version", dest="wf_version", type=str, required=True, help="workflow version")
-    parser.add_argument("-r", "--wf-run", dest="wf_run", type=str, required=True, help="workflow run ID")
+    parser.add_argument("-x", "--experiment-info-tsv",
+                        help="tsv file containing experiment information submitted from user")
+    parser.add_argument("-r", "--read-group-info-tsv",
+                        help="tsv file containing read_group information submitted from user")
+    parser.add_argument("-f", "--file-info-tsv",
+                        help="tsv file containing file information submitted from user")
     args = parser.parse_args()
 
-    main(args)
+    validate_args(args)
+
+    if args.metadata_json:
+        with open(args.metadata_json, 'r') as f:
+            metadata = json.load(f)
+    else:
+        # fistly TSV format conformity check, if not well-formed no point to continue
+        tsv_confomity_check('experiment', args.experiment_info_tsv)
+        tsv_confomity_check('read_group', args.read_group_info_tsv)
+        tsv_confomity_check('file', args.file_info_tsv)
+
+        # all TSV are well-formed, let's load them
+        metadata = load_all_tsvs(
+                            args.experiment_info_tsv,
+                            args.read_group_info_tsv,
+                            args.file_info_tsv
+                        )
+
+        # all TSV are well-formed, let's load them
+        metadata = load_all_tsvs(args.experiment_info_tsv, args.read_group_info_tsv, args.file_info_tsv)
+
+    main(metadata)

tools/payload-gen-seq-experiment/tests/checker.nf

@@ -23,20 +23,19 @@
 
 nextflow.enable.dsl=2
 
-params.user_submit_metadata = ""
-params.wf_name = ""
-params.wf_short_name = ""
-params.wf_version = ""
+params.metadata_json = "NO_FILE1"
+params.experiment_info_tsv = "NO_FILE2"
+params.read_group_info_tsv = "NO_FILE3"
+params.file_info_tsv = "NO_FILE4"
 
 include {payloadGenSeqExperiment} from "../payload-gen-seq-experiment" params(params)
 
 workflow {
   main:
     payloadGenSeqExperiment(
-      file(params.user_submit_metadata),
-      params.wf_name,
-      params.wf_short_name,
-      params.wf_version,
-      'ok'
+      file(params.metadata_json),
+      file(params.experiment_info_tsv),
+      file(params.read_group_info_tsv),
+      file(params.file_info_tsv)
     )
 }

tools/payload-gen-seq-experiment/tests/test-job-bam.nf.json

@@ -1,6 +1,3 @@
 {
-  "user_submit_metadata": "data/seq-exp.bam.metadata.json",
-  "wf_name": "sequencing-data-submission",
-  "wf_short_name": "seq-submission",
-  "wf_version": "0.1.0.0"
+  "metadata_json": "data/seq-exp.bam.metadata.json"
 }