add tool to parse metadata

Author: lindaxiang

json-parser/Dockerfile

@@ -2,10 +2,10 @@ FROM python:3.7.5-slim-buster
 
 LABEL org.opencontainers.image.source https://github.com/icgc-argo-workflows/data-processing-utility-tools
 
-ENV PATH="/tools:${PATH}"
+RUN apt-get update && apt-get install -y procps jq
 
-COPY *.py /tools/
+ENV PATH="/tools:${PATH}"
 
-ENTRYPOINT ["/usr/bin/env"]
+WORKDIR /tools
 
-CMD ["/bin/bash"]
+COPY . .

json-parser/main.nf

@@ -44,8 +44,7 @@ params.publish_dir = ""  // set to empty string will disable publishDir
 
 
 // tool specific parmas go here, add / change as needed
-params.input_file = ""
-params.output_pattern = "*"  // output file name pattern
+params.metadata_analysis = ""
 
 
 process jsonParser {
@@ -55,22 +54,29 @@ process jsonParser {
   cpus params.cpus
   memory "${params.mem} GB"
 
-  input:  // input, make update as needed
-    path input_file
+  input:
+    path metadata_analysis
 
-  output:  // output, make update as needed
-    path "output_dir/${params.output_pattern}", emit: output_file
+  output:
+    env STUDY_ID, emit: study_id
+    env DONOR_ID, emit: donor_id
+    env EXP, emit: experimental_strategy
+    env PAIRED, emit: paired
+    env ANALYSIS_TOOLS, emit: analysis_tools
+    env STRAND, emit: library_strandedness
 
   script:
-    // add and initialize variables here as needed
-
     """
-    mkdir -p output_dir
-
-    main.py \
-      -i ${input_file} \
-      -o output_dir
-
+    set -euxo pipefail
+    VARIABLE1=`cat ${metadata_analysis} | jq -r 'if ([.read_groups[]?] | length) >0 then [.read_groups[] | .is_paired_end] | all | tostring else null end' | tr -d '\\n'`
+    PAIRED=\${VARIABLE1:-'null'}
+    VARIABLE2=`cat ${metadata_analysis} | jq -r '[.files[] | .info? | .analysis_tools[]?] | unique | join(",")' | tr -d '\\n'`
+    ANALYSIS_TOOLS=\${VARIABLE2:-'null'}
+    VARIABLE3=`cat ${metadata_analysis} | jq -r '.experiment | .library_strandedness?' | tr -d '\\n'`
+    STRAND=\${VARIABLE3:-'null'}
+    STUDY_ID=`cat ${metadata_analysis} | jq -er '.studyId' | tr -d '\\n'`
+    DONOR_ID=`cat ${metadata_analysis} | jq -er '.samples[0].donor.donorId' | tr -d '\\n'`
+    EXP=`cat ${metadata_analysis} | jq -er '.experiment | .experimental_strategy?  // .library_strategy' | tr -d '\\n'`
     """
 }
 
@@ -79,6 +85,6 @@ process jsonParser {
 // using this command: nextflow run <git_acc>/<repo>/<pkg_name>/<main_script>.nf -r <pkg_name>.v<pkg_version> --params-file xxx
 workflow {
   jsonParser(
-    file(params.input_file)
+    file(params.metadata_analysis)
   )
 }

json-parser/main.py

@@ -1,11 +1,12 @@
 {
     "name": "json-parser",
     "version": "0.1.0",
-    "description": "python:3.7.5-slim-buster",
+    "description": "tool to parse JSON metadata",
     "main": "main.nf",
     "deprecated": false,
     "keywords": [
-        "tool to parse metadata"
+        "metadata",
+        "JSON"
     ],
     "repository": {
         "type": "git",

json-parser/pkg.json

@@ -43,60 +43,31 @@ params.container_version = ""
 params.container = ""
 
 // tool specific parmas go here, add / change as needed
-params.input_file = ""
+params.metadata_analysis = ""
 params.expected_output = ""
 
 include { jsonParser } from '../main'
 
-
-process file_smart_diff {
-  container "${params.container ?: container[params.container_registry ?: default_container_registry]}:${params.container_version ?: version}"
-
-  input:
-    path output_file
-    path expected_file
-
-  output:
-    stdout()
-
-  script:
-    """
-    # Note: this is only for demo purpose, please write your own 'diff' according to your own needs.
-    # in this example, we need to remove date field before comparison eg, <div id="header_filename">Tue 19 Jan 2021<br/>test_rg_3.bam</div>
-    # sed -e 's#"header_filename">.*<br/>test_rg_3.bam#"header_filename"><br/>test_rg_3.bam</div>#'
-
-    cat ${output_file[0]} \
-      | sed -e 's#"header_filename">.*<br/>#"header_filename"><br/>#' > normalized_output
-
-    ([[ '${expected_file}' == *.gz ]] && gunzip -c ${expected_file} || cat ${expected_file}) \
-      | sed -e 's#"header_filename">.*<br/>#"header_filename"><br/>#' > normalized_expected
-
-    diff normalized_output normalized_expected \
-      && ( echo "Test PASSED" && exit 0 ) || ( echo "Test FAILED, output file mismatch." && exit 1 )
-    """
-}
-
-
 workflow checker {
   take:
-    input_file
-    expected_output
+    metadata_analysis
 
   main:
     jsonParser(
-      input_file
-    )
-
-    file_smart_diff(
-      jsonParser.out.output_file,
-      expected_output
+      metadata_analysis
     )
+    jsonParser.out.study_id.set{study_id}
+    study_id.view()
+    jsonParser.out.donor_id.view()
+    jsonParser.out.experimental_strategy.view()
+    jsonParser.out.analysis_tools.view()
+    jsonParser.out.paired.view()
+    jsonParser.out.library_strandedness.view()
 }
 
 
 workflow {
   checker(
-    file(params.input_file),
-    file(params.expected_output)
+    file(params.metadata_analysis)
   )
 }

json-parser/tests/checker.nf

@@ -1,7 +1,5 @@
 {
-    "input_file": "input/test_rg_3.bam",
-    "expected_output": "expected/expected.test_rg_3.bam",
-    "publish_dir": "outdir",
-    "cpus": 1,
-    "mem": 0.5
+  "metadata_analysis": "data/40054341-119d-492e-8543-41119dc92ea1.sequencing_experiment.4.analysis.json",         
+  "cpus": 1,
+  "mem": 0.5
 }

json-parser/tests/input/README.md

@@ -1,8 +1,5 @@
 {
-    "input_file": "input/test_rg_3.bam",
-    "expected_output": "expected/expected.test_rg_3.bam",
-    "publish_dir": "outdir",
-    "container_registry": "ghcr.io",
-    "cpus": 1,
-    "mem": 0.5
+  "metadata_analysis": "data/89176bbc-ffca-4d73-984a-4a92ed931a98.sequencing_experiment.4.analysis.json",
+  "cpus": 1,
+  "mem": 0.5
 }

json-parser/tests/input/test_rg_3.bam

@@ -49,7 +49,8 @@
     "sequencing_date": "2014-12-12",
     "library_strategy": "WGS",
     "sequencing_center": "EXT",
-    "submitter_sequencing_experiment_id": "TEST_EXP"
+    "submitter_sequencing_experiment_id": "TEST_EXP",
+    "library_strandedness": "Unstrained"
   },
   "read_groups": [
     {