work in progress

Author: junjun-zhang

process/score_upload.nf

@@ -5,6 +5,9 @@ nextflow.enable.dsl=2
 params.cpus = 8
 params.mem = 20
 
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
+
 // required params w/ default
 params.container_version = "5.0.0"
 params.transport_mem = 2 // Transport memory is in number of GBs
@@ -17,6 +20,12 @@ params.api_token = "" // song/score API token for download process
 // --score_url        score url for download process
 
 process scoreUpload {
+    maxRetries params.max_retries
+    errorStrategy {
+        sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long);  // backoff time doubles before each retry
+        return params.max_retries ? 'retry' : 'finish'
+    }
+
     pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"]
 
     cpus params.cpus

process/song_manifest.nf

@@ -5,6 +5,9 @@ nextflow.enable.dsl=2
 params.cpus = 1
 params.mem = 1
 
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
+
 // required params w/ default
 params.container_version = "4.2.1"
 
@@ -16,6 +19,12 @@ params.api_token = "" // song/score API token for download process
 // --score_url        score url for download process
 
 process songManifest {
+    maxRetries params.max_retries
+    errorStrategy {
+        sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long);  // backoff time doubles before each retry
+        return params.max_retries ? 'retry' : 'finish'
+    }
+
     pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"]
 
     cpus params.cpus

process/song_publish.nf

@@ -5,6 +5,9 @@ nextflow.enable.dsl=2
 params.cpus = 1
 params.mem = 1
 
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
+
 // required params w/ default
 params.container_version = "4.2.1"
 
@@ -16,6 +19,12 @@ params.api_token = "" // song/score API token for download process
 // --score_url        score url for download process
 
 process songPublish {
+    maxRetries params.max_retries
+    errorStrategy {
+        sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long);  // backoff time doubles before each retry
+        return params.max_retries ? 'retry' : 'finish'
+    }
+
     pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"]
 
     cpus params.cpus

process/song_submit.nf

@@ -5,6 +5,9 @@ nextflow.enable.dsl=2
 params.cpus = 1
 params.mem = 1
 
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
+
 // required params w/ default
 params.container_version = "4.2.1"
 
@@ -16,6 +19,12 @@ params.api_token = "" // song/score API token for download process
 // --score_url        score url for download process
 
 process songSubmit {
+    maxRetries params.max_retries
+    errorStrategy {
+        sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long);  // backoff time doubles before each retry
+        return params.max_retries ? 'retry' : 'finish'
+    }
+
     pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"]
 
     cpus params.cpus

…ore-upload/local_modules/score_upload.nf …ore-upload/local_modules/score-upload.nfsong-score-upload/local_modules/score_upload.nf renamed to song-score-upload/local_modules/score-upload.nf song-score-upload/local_modules/score_upload.nf renamed to song-score-upload/local_modules/score-upload.nf

@@ -16,6 +16,7 @@
   limitations under the License.
 
   Authors:
+    Alex Lepsa
     Junjun Zhang
 */
 
@@ -25,37 +26,88 @@ version = '2.6.0'  // package version
 // universal params go here, change default value as needed
 params.container = ""
 params.container_registry = ""
-params.container_version = ""
-params.cpus = 1
-params.mem = 1  // GB
 params.publish_dir = ""  // set to empty string will disable publishDir
 
-// tool specific parmas go here, add / change as needed
-params.input_file = ""
-params.cleanup = true
-
-
-
-// please update workflow code as needed
-workflow SongScoreUpload {
-  take:  // update as needed
-    input_file
-
-
-  main:  // update as needed
-    demoCopyFile(input_file)
-    if (params.cleanup) { cleanupWorkdir(demoCopyFile.out, true) }
-
-  emit:  // update as needed
-    output_file = demoCopyFile.out.output_file
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
 
+// tool specific parmas go here, add / change as needed
+params.study_id = "TEST-PR"
+params.payload = "NO_FILE"
+params.upload = []
+
+params.api_token = ""
+
+params.song_cpus = 1
+params.song_mem = 1  // GB
+params.song_url = "https://song.rdpc-qa.cancercollaboratory.org"
+params.song_api_token = ""
+params.song_container_version = "4.2.1"
+
+params.score_cpus = 1
+params.score_mem = 1  // GB
+params.score_transport_mem = 1  // GB
+params.score_url = "https://score.rdpc-qa.cancercollaboratory.org"
+params.score_api_token = ""
+params.score_container_version = "5.0.0"
+
+
+song_params = [
+    *:params,
+    'cpus': params.song_cpus,
+    'mem': params.song_mem,
+    'song_url': params.song_url,
+    'song_container_version': params.song_container_version,
+    'api_token': params.song_api_token ?: params.api_token,
+    'publish_dir': ''
+]
+
+score_params = [
+    *:params,
+    'cpus': params.score_cpus,
+    'mem': params.score_mem,
+    'transport_mem': params.score_transport_mem,
+    'song_url': params.song_url,
+    'score_url': params.score_url,
+    'score_container_version': params.score_container_version,
+    'api_token': params.score_api_token ?: params.api_token
+]
+
+include { songSubmit } from './local_modules/song-submit' params(song_params)
+include { songManifest } from './local_modules/song-manifest' params(song_params)
+include { scoreUpload } from './local_modules/score-upload' params(song_params)
+include { songPublish } from './local_modules/song-publish' params(song_params)
+
+
+workflow songScoreUpload {
+    take: study_id
+    take: payload
+    take: upload
+
+    main:
+        // Create new analysis
+        songSubmit(study_id, payload)
+
+        // Generate file manifest for upload
+        songManifest(study_id, songSubmit.out, upload)
+
+        // Upload to SCORE
+        scoreUpload(songSubmit.out, songManifest.out, upload)
+
+        // Publish the analysis
+        songPublish(study_id, scoreUpload.out.ready_to_publish)
+
+    emit:
+        analysis_id = songPublish.out.analysis_id
 }
 
 
 // this provides an entry point for this main script, so it can be run directly without clone the repo
 // using this command: nextflow run <git_acc>/<repo>/<pkg_name>/<main_script>.nf -r <pkg_name>.v<pkg_version> --params-file xxx
 workflow {
   SongScoreUpload(
-    file(params.input_file)
+    params.study_id,
+    file(params.payload),
+    Channel.fromPath(params.upload)
   )
-}
+}

…re-upload/local_modules/song_manifest.nf …re-upload/local_modules/song-manifest.nfsong-score-upload/local_modules/song_manifest.nf renamed to song-score-upload/local_modules/song-manifest.nf song-score-upload/local_modules/song_manifest.nf renamed to song-score-upload/local_modules/song-manifest.nf

@@ -27,63 +27,79 @@
 nextflow.enable.dsl = 2
 version = '0.2.6'  // package version
 
-// universal params
-params.publish_dir = ""
+// universal params go here, change default value as needed
 params.container = ""
 params.container_registry = ""
-params.container_version = ""
+params.publish_dir = ""  // set to empty string will disable publishDir
+
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
 
 // tool specific parmas go here, add / change as needed
-params.input_file = ""
-params.expected_output = ""
-params.cleanup = false
+params.study_id = "TEST-PR"
+params.payload = "NO_FILE"
+params.upload = []
+
+params.api_token = ""
+
+params.song_cpus = 1
+params.song_mem = 1  // GB
+params.song_url = "https://song.rdpc-qa.cancercollaboratory.org"
+params.song_api_token = ""
+params.song_container_version = "4.2.1"
+
+params.score_cpus = 1
+params.score_mem = 1  // GB
+params.score_transport_mem = 1  // GB
+params.score_url = "https://score.rdpc-qa.cancercollaboratory.org"
+params.score_api_token = ""
+params.score_container_version = "5.0.0"
+
+
+song_params = [
+    *:params,
+    'cpus': params.song_cpus,
+    'mem': params.song_mem,
+    'song_url': params.song_url,
+    'song_container_version': params.song_container_version,
+    'api_token': params.song_api_token ?: params.api_token,
+    'publish_dir': ''
+]
+
+score_params = [
+    *:params,
+    'cpus': params.score_cpus,
+    'mem': params.score_mem,
+    'transport_mem': params.score_transport_mem,
+    'song_url': params.song_url,
+    'score_url': params.score_url,
+    'score_container_version': params.score_container_version,
+    'api_token': params.score_api_token ?: params.api_token
+]
 
 include { SongScoreUpload } from '../main'
-// include section starts
-// include section ends
-
-
-process file_smart_diff {
-  input:
-    path output_file
-    path expected_file
-
-  output:
-    stdout()
-
-  script:
-    """
-    # Note: this is only for demo purpose, please write your own 'diff' according to your own needs.
-    # remove date field before comparison eg, <div id="header_filename">Tue 19 Jan 2021<br/>test_rg_3.bam</div>
-    # sed -e 's#"header_filename">.*<br/>test_rg_3.bam#"header_filename"><br/>test_rg_3.bam</div>#'
-
-    diff <( cat ${output_file} | sed -e 's#"header_filename">.*<br/>#"header_filename"><br/>#' ) \
-         <( ([[ '${expected_file}' == *.gz ]] && gunzip -c ${expected_file} || cat ${expected_file}) | sed -e 's#"header_filename">.*<br/>#"header_filename"><br/>#' ) \
-    && ( echo "Test PASSED" && exit 0 ) || ( echo "Test FAILED, output file mismatch." && exit 1 )
-    """
-}
 
 
 workflow checker {
   take:
-    input_file
-    expected_output
+    study_id
+    payload
+    upload
 
   main:
     SongScoreUpload(
-      input_file
+      study_id,
+      payload,
+      upload
     )
 
-    file_smart_diff(
-      SongScoreUpload.out.output_file,
-      expected_output
-    )
 }
 
 
 workflow {
   checker(
-    file(params.input_file),
-    file(params.expected_output)
+    params.study_id,
+    file(params.payload),
+    Channel.fromPath(params.upload)
   )
 }