add tool payload-gen-variant-filtering

Author: lindaxiang

payload-gen-variant-filtering/main.nf

@@ -17,7 +17,7 @@
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
   Authors:
-    lindaxiang
+    Linda Xiang (linda.xiang@oicr.on.ca)
 */
 
 /********************************************************************/
@@ -44,8 +44,11 @@ params.publish_dir = ""  // set to empty string will disable publishDir
 
 
 // tool specific parmas go here, add / change as needed
-params.input_file = ""
-params.output_pattern = "*"  // output file name pattern
+params.analysis = ""
+params.files_to_upload = []
+params.wf_name = ""
+params.wf_short_name = ""
+params.wf_version = ""
 
 
 process payloadGenVariantFiltering {
@@ -56,10 +59,14 @@ process payloadGenVariantFiltering {
   memory "${params.mem} GB"
 
   input:  // input, make update as needed
-    path input_file
+    path analysis
+    path files_to_upload
+    val wf_name
+    val wf_short_name
+    val wf_version
 
   output:  // output, make update as needed
-    path "output_dir/${params.output_pattern}", emit: output_file
+    path "*.payload.json", emit: payload
 
   script:
     // add and initialize variables here as needed
@@ -68,8 +75,13 @@ process payloadGenVariantFiltering {
     mkdir -p output_dir
 
     main.py \
-      -i ${input_file} \
-      -o output_dir
+      -a ${analysis} \
+      -f ${files_to_upload} \
+      -w ${wf_name} \
+      -s ${wf_short_name} \
+      -v ${wf_version} \
+      -r ${workflow.runName} \
+      -j ${workflow.sessionId}
 
     """
 }
@@ -79,6 +91,10 @@ process payloadGenVariantFiltering {
 // using this command: nextflow run <git_acc>/<repo>/<pkg_name>/<main_script>.nf -r <pkg_name>.v<pkg_version> --params-file xxx
 workflow {
   payloadGenVariantFiltering(
-    file(params.input_file)
+    file(params.analysis),
+    Channel.fromPath(params.files_to_upload).collect(),
+    params.wf_name,
+    params.wf_short_name,
+    params.wf_version
   )
 }

payload-gen-variant-filtering/main.py

@@ -18,36 +18,124 @@
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
   Authors:
-    lindaxiang
+    Linda Xiang (linda.xiang@oicr.on.ca)
 """
 
 import os
-import sys
 import argparse
-import subprocess
+import json
+import uuid
+import hashlib
+import copy
 
+variant_type_to_data_type_etc = {
+    'snv': ['Simple Nucleotide Variation', 'Raw SNV Calls', ['CaVEMan', 'bcftools'], ['GATK-Mutect2', 'bcftools']],   # dataCategory, dataType, analysis_tools
+    'indel': ['Simple Nucleotide Variation', 'Raw InDel Calls', ['Pindel', 'bcftools'], ['GATK-Mutect2', 'bcftools']]
+}
+
+def calculate_size(file_path):
+    return os.stat(file_path).st_size
+
+
+def calculate_md5(file_path):
+    md5 = hashlib.md5()
+    with open(file_path, 'rb') as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b''):
+            md5.update(chunk)
+    return md5.hexdigest()
+
+
+def get_files_info(file_to_upload):
+    basename = os.path.basename(file_to_upload)
+    input_wf = basename.split(".")[5]
+    variant_type = basename.split(".")[8]
+    file_info = {
+        'fileName': basename,
+        'fileType': 'VCF' if basename.endswith('.vcf.gz') else basename.split(".")[-1].upper(),
+        'fileSize': calculate_size(file_to_upload),
+        'fileMd5sum': calculate_md5(file_to_upload),
+        'fileAccess': 'open',
+        'info': {
+            'data_category': variant_type_to_data_type_etc[variant_type][0]
+        }
+    }
+
+    if file_to_upload.endswith('.vcf.gz'):
+        file_info['dataType'] = variant_type_to_data_type_etc[variant_type][1]
+    elif file_to_upload.endswith('.vcf.gz.tbi'):
+        file_info['dataType'] = 'VCF Index'
+    else:
+        pass
+
+    if input_wf in (['sanger-wgs', 'sanger-wxs']):
+        file_info['info']['analysis_tools'] = variant_type_to_data_type_etc[variant_type][2]
+    elif input_wf in (['gatk-mutect2']):
+        file_info['info']['analysis_tools'] = variant_type_to_data_type_etc[variant_type][3]
+
+    return file_info
+
+def get_sample_info(sample_list):
+    samples = copy.deepcopy(sample_list)
+    for sample in samples:
+        for item in ['info', 'sampleId', 'specimenId', 'donorId', 'studyId']:
+            sample.pop(item, None)
+            sample['specimen'].pop(item, None)
+            sample['donor'].pop(item, None)
+
+    return samples
 
 def main():
     """
     Python implementation of tool: payload-gen-variant-filtering
-
-    This is auto-generated Python code, please update as needed!
     """
 
     parser = argparse.ArgumentParser(description='Tool: payload-gen-variant-filtering')
-    parser.add_argument('-i', '--input-file', dest='input_file', type=str,
-                        help='Input file', required=True)
-    parser.add_argument('-o', '--output-dir', dest='output_dir', type=str,
-                        help='Output directory', required=True)
+    parser.add_argument("-a", dest="analysis", required=True,
+                        help="json file containing sequencing_alignment SONG analysis for tumour sample")
+    parser.add_argument("-f", dest="files_to_upload", type=str, nargs="+", help="Files to be uploaded", required=True)
+    parser.add_argument("-w", dest="wf_name", type=str, help="workflow full name", required=True)
+    parser.add_argument("-s", dest="wf_short_name", type=str, help="workflow short name", required=True)
+    parser.add_argument("-v", dest="wf_version", type=str, required=True, help="workflow version")
+    parser.add_argument("-r", dest="wf_run", type=str, required=True, help="workflow run ID")
+    parser.add_argument("-j", dest="wf_session", type=str, required=True, help="workflow session ID")
     args = parser.parse_args()
 
-    if not os.path.isfile(args.input_file):
-        sys.exit('Error: specified input file %s does not exist or is not accessible!' % args.input_file)
+    analysis = {}
+    with open(args.analysis, 'r') as f:
+        analysis = json.load(f)
+
+    analysis_type = analysis.get('analysisType').get('name')
+    payload = {
+        'analysisType': {
+            'name': analysis_type
+        },
+        'studyId': analysis.get('studyId'),  
+        'experiment': analysis.get('experiment'),
+        'samples': get_sample_info(analysis.get('samples')),
+        'files': [],
+        'workflow': {
+            'workflow_name': 'Open Access Variant Filtering',
+            'workflow_short_name': args.wf_short_name,
+            'workflow_version': args.wf_version,
+            'run_id': args.wf_run,
+            'session_id': args.wf_session,
+            'inputs': [
+                {
+                    'input_analysis_id': analysis.get('analysisId'),
+                    'analysis_type': analysis_type
+                }
+            ],
+            'genome_build': 'GRCh38_hla_decoy_ebv'
+        },
+        'variant_class': analysis.get('variant_class')
+    }
 
-    if not os.path.isdir(args.output_dir):
-        sys.exit('Error: specified output dir %s does not exist or is not accessible!' % args.output_dir)
+    for f in args.files_to_upload:
+        file_info = get_files_info(f)
+        payload['files'].append(file_info)
 
-    subprocess.run(f"cp {args.input_file} {args.output_dir}/", shell=True, check=True)
+    with open("%s.%s.payload.json" % (str(uuid.uuid4()), analysis_type), 'w') as f:
+        f.write(json.dumps(payload, indent=2))
 
 
 if __name__ == "__main__":

payload-gen-variant-filtering/payload-gen-variant-filtering.nf

@@ -17,7 +17,7 @@
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
   Authors:
-    lindaxiang
+    Linda Xiang (linda.xiang@oicr.on.ca)
 */
 
 /*
@@ -43,16 +43,15 @@ params.container_version = ""
 params.container = ""
 
 // tool specific parmas go here, add / change as needed
-params.input_file = ""
+params.analysis = ""
+params.files_to_upload = []
+params.wf_name = ""
+params.wf_short_name = ""
+params.wf_version = ""
 params.expected_output = ""
 
 include { payloadGenVariantFiltering } from '../main'
 
-Channel
-  .fromPath(params.input_file, checkIfExists: true)
-  .set { input_file }
-
-
 process file_smart_diff {
   container "${params.container ?: container[params.container_registry ?: default_container_registry]}:${params.container_version ?: version}"
 
@@ -65,37 +64,45 @@ process file_smart_diff {
 
   script:
     """
-    # Note: this is only for demo purpose, please write your own 'diff' according to your own needs.
-    # remove date field before comparison eg, <div id="header_filename">Tue 19 Jan 2021<br/>test_rg_3.bam</div>
-    # sed -e 's#"header_filename">.*<br/>test_rg_3.bam#"header_filename"><br/>test_rg_3.bam</div>#'
-
-    diff <( cat ${output_file} | sed -e 's#"header_filename">.*<br/>#"header_filename"><br/>#' ) \
-         <( ([[ '${expected_file}' == *.gz ]] && gunzip -c ${expected_file} || cat ${expected_file}) | sed -e 's#"header_filename">.*<br/>#"header_filename"><br/>#' ) \
+    diff <( cat ${output_file} |sort | sed '/\"run_id\"/d' | sed '/\"session_id\"/d' ) \
+         <( cat ${expected_file} |sort | sed '/\"run_id\"/d' | sed '/\"session_id\"/d' ) \
     && ( echo "Test PASSED" && exit 0 ) || ( echo "Test FAILED, output file mismatch." && exit 1 )
     """
 }
 
 
 workflow checker {
   take:
-    input_file
+    analysis
+    files_to_upload
+    wf_name
+    wf_short_name
+    wf_version
     expected_output
 
   main:
     payloadGenVariantFiltering(
-      input_file
+      analysis,
+      files_to_upload,
+      wf_name,
+      wf_short_name,
+      wf_version
     )
 
     file_smart_diff(
-      payloadGenVariantFiltering.out.output_file,
+      payloadGenVariantFiltering.out.payload,
       expected_output
     )
 }
 
 
 workflow {
   checker(
-    file(params.input_file),
+    file(params.analysis),
+    Channel.fromPath(params.files_to_upload).collect(),
+    params.wf_name,
+    params.wf_short_name,
+    params.wf_version,
     file(params.expected_output)
   )
 }