address the PR comments

Author: lindaxiang

payload-gen-variant-processing/Dockerfile

@@ -2,6 +2,8 @@ FROM python:3.7.5-slim-buster
 
 LABEL org.opencontainers.image.source https://github.com/icgc-argo/data-processing-utility-tools
 
+RUN apt-get update && apt-get install -y procps jq
+
 RUN groupadd -g 1000 ubuntu &&\
     useradd -l -u 1000 -g ubuntu ubuntu &&\
     install -d -m 0755 -o ubuntu -g ubuntu /home/ubuntu

payload-gen-variant-processing/main.nf

@@ -49,9 +49,7 @@ params.files_to_upload = []
 params.wf_name = ""
 params.wf_short_name = ""
 params.wf_version = ""
-params.controlled = false
-params.process_name = ""
-params.analysis_type = ""
+params.open = false
 
 
 process payloadGenVariantProcessing {
@@ -67,19 +65,15 @@ process payloadGenVariantProcessing {
     val wf_name
     val wf_short_name
     val wf_version
-    val controlled
-    val process_name
-    val analysis_type
+    val open
 
   output:  // output, make update as needed
     path "*.payload.json", emit: payload
     path "out/*{.vcf.gz,.vcf.gz.tbi}", emit: files_to_upload
 
   script:
     // add and initialize variables here as needed
-    arg_controlled = controlled ? "-c" : ""
-    arg_process_name = process_name ? "-p ${process_name}" : ""
-    arg_analysis_type = analysis_type ? "-t ${analysis_type}" : ""
+    arg_open = open ? "-o" : ""
 
     """   
     main.py \
@@ -90,7 +84,7 @@ process payloadGenVariantProcessing {
       -v ${wf_version} \
       -r ${workflow.runName} \
       -j ${workflow.sessionId} \
-      ${arg_controlled} ${arg_process_name} ${arg_analysis_type}
+      ${arg_open}
 
     """
 }
@@ -105,8 +99,6 @@ workflow {
     params.wf_name,
     params.wf_short_name,
     params.wf_version,
-    params.controlled,
-    params.process_name,
-    params.analysis_type
+    params.open
   )
 }

payload-gen-variant-processing/main.py

@@ -53,14 +53,14 @@ def get_files_info(file_to_upload, args):
     fname = os.path.basename(file_to_upload).split(".")
     input_wf = fname[5]
     variant_type = fname[7]
-    process_name = args.process_name if args.process_name else args.wf_short_name
+    process_name = args.wf_short_name
     new_fname = ".".join(fname[0:8]+[process_name, 'vcf.gz']+(['tbi'] if file_to_upload.endswith('.tbi') else []))
     file_info = {
         'fileName': new_fname,
         'fileType': 'VCF' if new_fname.endswith('.vcf.gz') else new_fname.split(".")[-1].upper(),
         'fileSize': calculate_size(file_to_upload),
         'fileMd5sum': calculate_md5(file_to_upload),
-        'fileAccess': 'open' if not args.controlled else 'controlled',
+        'fileAccess': 'open' if args.open else 'controlled',
         'info': {
             'data_category': variant_type_to_data_type_etc[variant_type][0]
         }
@@ -85,7 +85,6 @@ def get_files_info(file_to_upload, args):
         pass
 
     dst = os.path.join(os.getcwd(), new_dir, new_fname)
-    print(dst)
     os.symlink(os.path.abspath(file_to_upload), dst)
 
     return file_info
@@ -114,20 +113,18 @@ def main():
     parser.add_argument("-v", dest="wf_version", type=str, required=True, help="workflow version")
     parser.add_argument("-r", dest="wf_run", type=str, required=True, help="workflow run ID")
     parser.add_argument("-j", dest="wf_session", type=str, required=True, help="workflow session ID")
-    parser.add_argument("-c", dest="controlled", action='store_true', help="set file to be controlled access")
-    parser.add_argument("-p", dest="process_name", type=str, help="variant process name")
-    parser.add_argument("-t", dest="analysis_type", type=str, default="variant_processing", help="specify output song analysis type")
+    parser.add_argument("-o", dest="open", action='store_true', help="set file to be open access")
     args = parser.parse_args()
 
     analysis = {}
     with open(args.analysis, 'r') as f:
         analysis = json.load(f)
 
     input_analysis_type = analysis.get('analysisType').get('name')
-    
+    analysis_type = "variant_processing"
     payload = {
         'analysisType': {
-            'name': args.analysis_type
+            'name': analysis_type
         },
         'studyId': analysis.get('studyId'),  
         'experiment': analysis.get('experiment'),
@@ -154,7 +151,7 @@ def main():
         file_info = get_files_info(f, args)
         payload['files'].append(file_info)
 
-    with open("%s.%s.payload.json" % (str(uuid.uuid4()), args.analysis_type), 'w') as f:
+    with open("%s.%s.payload.json" % (str(uuid.uuid4()), analysis_type), 'w') as f:
         f.write(json.dumps(payload, indent=2))
 
 

payload-gen-variant-processing/tests/checker.nf

@@ -48,9 +48,7 @@ params.files_to_upload = []
 params.wf_name = ""
 params.wf_short_name = ""
 params.wf_version = ""
-params.controlled = false
-params.process_name = ""
-params.analysis_type = ""
+params.open = false
 params.expected_output = ""
 
 include { payloadGenVariantProcessing } from '../main'
@@ -67,8 +65,8 @@ process file_smart_diff {
 
   script:
     """
-    diff <( cat ${output_file} |sort | sed '/\"run_id\"/d' | sed '/\"session_id\"/d' ) \
-         <( cat ${expected_file} |sort | sed '/\"run_id\"/d' | sed '/\"session_id\"/d' ) \
+    diff <( cat ${output_file} | jq -S . | sed '/\"run_id\"/d' | sed '/\"session_id\"/d' ) \
+         <( cat ${expected_file} | jq -S . | sed '/\"run_id\"/d' | sed '/\"session_id\"/d' ) \
     && ( echo "Test PASSED" && exit 0 ) || ( echo "Test FAILED, output file mismatch." && exit 1 )
     """
 }
@@ -81,9 +79,7 @@ workflow checker {
     wf_name
     wf_short_name
     wf_version
-    controlled
-    process_name
-    analysis_type
+    open
     expected_output
 
   main:
@@ -93,9 +89,7 @@ workflow checker {
       wf_name,
       wf_short_name,
       wf_version,
-      controlled,
-      process_name,
-      analysis_type
+      open
     )
 
     file_smart_diff(
@@ -112,9 +106,7 @@ workflow {
     params.wf_name,
     params.wf_short_name,
     params.wf_version,
-    params.controlled,
-    params.process_name,
-    params.analysis_type,
+    params.open,
     file(params.expected_output)
   )
 }

payload-gen-variant-processing/tests/expected/expected.sanger-wgs.filtered.snv.payload.json

@@ -1,6 +1,6 @@
 {
   "analysisType": {
-    "name": "variant_annotating"
+    "name": "variant_processing"
   },
   "studyId": "TEST-PR",
   "experiment": {
@@ -26,7 +26,7 @@
   ],
   "files": [
     {
-      "fileName": "TEST-PR.DO250183.SA610229.wgs.20200513.sanger-wgs.somatic.snv.annotation.vcf.gz",
+      "fileName": "TEST-PR.DO250183.SA610229.wgs.20200513.sanger-wgs.somatic.snv.open-filter.vcf.gz",
       "fileType": "VCF",
       "fileSize": 23947,
       "fileMd5sum": "1ebce71c8231aeb7ac6c06c92adfd5db",
@@ -41,7 +41,7 @@
       "dataType": "Raw SNV Calls"
     },
     {
-      "fileName": "TEST-PR.DO250183.SA610229.wgs.20200513.sanger-wgs.somatic.snv.annotation.vcf.gz.tbi",
+      "fileName": "TEST-PR.DO250183.SA610229.wgs.20200513.sanger-wgs.somatic.snv.open-filter.vcf.gz.tbi",
       "fileType": "TBI",
       "fileSize": 185,
       "fileMd5sum": "7a8482dc43849a4ee8bdd4c9c77fae0c",
@@ -60,8 +60,8 @@
     "workflow_name": "Open Access Variant Filtering",
     "workflow_short_name": "open-filter",
     "workflow_version": "0.1.0",
-    "run_id": "hopeful_wescoff",
-    "session_id": "6f1137f4-7e04-4758-bf15-53936d76a1d3",
+    "run_id": "insane_rubens",
+    "session_id": "b5dbc2b3-f915-4f56-acbb-97d31df6c1f0",
     "inputs": [
       {
         "input_analysis_id": "3414534a-47a4-492d-9453-4a47a4192d6d",

payload-gen-variant-processing/tests/test-job-1.json

@@ -8,9 +8,6 @@
     "wf_name": "open-access-variant-filtering",
     "wf_short_name": "open-filter",
     "wf_version": "0.1.0",
-    "controlled": true,
-    "process_name": "annotation",
-    "analysis_type": "variant_annotating",
     "publish_dir": "outdir",
     "cpus": 1,
     "mem": 0.5

payload-gen-variant-processing/tests/test-job-2.json

@@ -8,6 +8,7 @@
     "wf_name": "open-access-variant-filtering",
     "wf_short_name": "open-filter",
     "wf_version": "0.1.0",
+    "open": true,
     "publish_dir": "outdir",
     "cpus": 1,
     "mem": 0.5

payload-gen-variant-processing/tests/test-job-3.json

@@ -8,6 +8,7 @@
     "wf_name": "open-access-variant-filtering",
     "wf_short_name": "open-filter",
     "wf_version": "0.1.0",
+    "open": true,
     "publish_dir": "outdir",
     "cpus": 1,
     "mem": 0.5