Merge pull request #32 from icgc-argo/song-score-upload@2.6.0

[release]

Author: junjun-zhang

process/score_upload.nf

@@ -5,6 +5,9 @@ nextflow.enable.dsl=2
 params.cpus = 8
 params.mem = 20
 
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
+
 // required params w/ default
 params.container_version = "5.0.0"
 params.transport_mem = 2 // Transport memory is in number of GBs
@@ -17,6 +20,12 @@ params.api_token = "" // song/score API token for download process
 // --score_url        score url for download process
 
 process scoreUpload {
+    maxRetries params.max_retries
+    errorStrategy {
+        sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long);  // backoff time doubles before each retry
+        return params.max_retries ? 'retry' : 'finish'
+    }
+
     pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"]
 
     cpus params.cpus

process/song_manifest.nf

@@ -5,6 +5,9 @@ nextflow.enable.dsl=2
 params.cpus = 1
 params.mem = 1
 
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
+
 // required params w/ default
 params.container_version = "4.2.1"
 
@@ -16,6 +19,12 @@ params.api_token = "" // song/score API token for download process
 // --score_url        score url for download process
 
 process songManifest {
+    maxRetries params.max_retries
+    errorStrategy {
+        sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long);  // backoff time doubles before each retry
+        return params.max_retries ? 'retry' : 'finish'
+    }
+
     pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"]
 
     cpus params.cpus

process/song_publish.nf

@@ -5,6 +5,9 @@ nextflow.enable.dsl=2
 params.cpus = 1
 params.mem = 1
 
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
+
 // required params w/ default
 params.container_version = "4.2.1"
 
@@ -16,6 +19,12 @@ params.api_token = "" // song/score API token for download process
 // --score_url        score url for download process
 
 process songPublish {
+    maxRetries params.max_retries
+    errorStrategy {
+        sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long);  // backoff time doubles before each retry
+        return params.max_retries ? 'retry' : 'finish'
+    }
+
     pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"]
 
     cpus params.cpus

process/song_submit.nf

@@ -5,6 +5,9 @@ nextflow.enable.dsl=2
 params.cpus = 1
 params.mem = 1
 
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
+
 // required params w/ default
 params.container_version = "4.2.1"
 
@@ -16,6 +19,12 @@ params.api_token = "" // song/score API token for download process
 // --score_url        score url for download process
 
 process songSubmit {
+    maxRetries params.max_retries
+    errorStrategy {
+        sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long);  // backoff time doubles before each retry
+        return params.max_retries ? 'retry' : 'finish'
+    }
+
     pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"]
 
     cpus params.cpus

song-score-upload/local_modules/score-upload.nf

@@ -0,0 +1 @@
+../../process/score_upload.nf

song-score-upload/local_modules/song-manifest.nf

@@ -0,0 +1 @@
+../../process/song_manifest.nf

song-score-upload/local_modules/song-publish.nf

@@ -0,0 +1 @@
+../../process/song_publish.nf

song-score-upload/local_modules/song-submit.nf

@@ -0,0 +1 @@
+../../process/song_submit.nf

song-score-upload/main.nf

@@ -0,0 +1,110 @@
+#!/usr/bin/env nextflow
+
+/*
+  Copyright (c) 2020-2021, Ontario Institute for Cancer Research
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Authors:
+    Alex Lepsa
+    Junjun Zhang
+*/
+
+nextflow.enable.dsl = 2
+version = '2.6.0'  // package version
+
+// universal params go here, change default value as needed
+params.max_retries = 5  // set to 0 will disable retry
+params.first_retry_wait_time = 1  // in seconds
+
+// tool specific parmas go here, add / change as needed
+params.study_id = "TEST-PR"
+params.payload = "NO_FILE"
+params.upload = []
+
+params.api_token = ""
+
+params.song_cpus = 1
+params.song_mem = 1  // GB
+params.song_url = "https://song.rdpc-qa.cancercollaboratory.org"
+params.song_api_token = ""
+params.song_container_version = "4.2.1"
+
+params.score_cpus = 1
+params.score_mem = 1  // GB
+params.score_transport_mem = 1  // GB
+params.score_url = "https://score.rdpc-qa.cancercollaboratory.org"
+params.score_api_token = ""
+params.score_container_version = "5.0.0"
+
+
+song_params = [
+    *:params,
+    'cpus': params.song_cpus,
+    'mem': params.song_mem,
+    'song_url': params.song_url,
+    'song_container_version': params.song_container_version,
+    'api_token': params.song_api_token ?: params.api_token
+]
+
+score_params = [
+    *:params,
+    'cpus': params.score_cpus,
+    'mem': params.score_mem,
+    'transport_mem': params.score_transport_mem,
+    'song_url': params.song_url,
+    'score_url': params.score_url,
+    'score_container_version': params.score_container_version,
+    'api_token': params.score_api_token ?: params.api_token
+]
+
+include { songSubmit as songSub } from './local_modules/song-submit' params(song_params)
+include { songManifest as songMan } from './local_modules/song-manifest' params(song_params)
+include { scoreUpload as scoreUp } from './local_modules/score-upload' params(score_params)
+include { songPublish as songPub } from './local_modules/song-publish' params(song_params)
+
+
+workflow SongScoreUpload {
+    take:
+        study_id
+        payload
+        upload
+
+    main:
+        // Create new analysis
+        songSub(study_id, payload)
+
+        // Generate file manifest for upload
+        songMan(study_id, songSub.out, upload.collect())
+
+        // Upload to SCORE
+        scoreUp(songSub.out, songMan.out, upload.collect())
+
+        // Publish the analysis
+        songPub(study_id, scoreUp.out.ready_to_publish)
+
+    emit:
+        analysis_id = songPub.out.analysis_id
+}
+
+
+// this provides an entry point for this main script, so it can be run directly without clone the repo
+// using this command: nextflow run <git_acc>/<repo>/<pkg_name>/<main_script>.nf -r <pkg_name>.v<pkg_version> --params-file xxx
+
+workflow {
+  SongScoreUpload(
+    params.study_id,
+    file(params.payload),
+    Channel.fromPath(params.upload)
+  )
+}

song-score-upload/nextflow.config

@@ -0,0 +1,4 @@
+docker {
+    enabled = true
+    runOptions = '-u \$(id -u):\$(id -g)'
+}