Skip to content

Commit 579619c

Browse files
junjun-zhangjunjun-zhang
authored
Merge pull request #31 from icgc-argo/song-score-download@2.6.0
[release]
2 parents a27566c + 21f9252 commit 579619c

14 files changed

Lines changed: 255 additions & 0 deletions

‎process/score_download.nf‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ nextflow.enable.dsl=2
55
params.cpus = 8
66
params.mem = 20
77

8+
params.publish_dir = ""
9+
10+
params.max_retries = 5 // set to 0 will disable retry
11+
params.first_retry_wait_time = 1 // in seconds
12+
813
// required params w/ default
914
params.container_version = "5.0.0"
1015
params.transport_mem = 2 // Transport memory is in number of GBs
@@ -18,12 +23,19 @@ params.api_token = "" // song/score API token for download process
1823

1924
// TODO: Replace with score container once it can download files via analysis_id
2025
process scoreDownload {
26+
maxRetries params.max_retries
27+
errorStrategy {
28+
sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long); // backoff time increases exponentially before each retry
29+
return params.max_retries ? 'retry' : 'finish'
30+
}
31+
2132
pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"]
2233

2334
cpus params.cpus
2435
memory "${params.mem} GB"
2536

2637
container "overture/score:${params.container_version}"
38+
publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir
2739

2840
label "scoreDownload"
2941
tag "${analysis_id}"

‎process/song_get_analysis.nf‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ nextflow.enable.dsl=2
55
params.cpus = 1
66
params.mem = 1
77

8+
params.publish_dir = ""
9+
10+
params.max_retries = 5 // set to 0 will disable retry
11+
params.first_retry_wait_time = 1 // in seconds
12+
813
// required params w/ default
914
params.container_version = "4.2.1"
1015

@@ -16,12 +21,19 @@ params.api_token = "" // song/score API token for download process
1621
// --score_url score url for download process
1722

1823
process songGetAnalysis {
24+
maxRetries params.max_retries
25+
errorStrategy {
26+
sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long); // backoff time increases exponentially before each retry
27+
return params.max_retries ? 'retry' : 'finish'
28+
}
29+
1930
pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"]
2031

2132
cpus params.cpus
2233
memory "${params.mem} GB"
2334

2435
container "overture/song-client:${params.container_version}"
36+
publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir
2537

2638
tag "${analysis_id}"
2739

‎song-score-download/local_modules/score-download.nf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../process/score_download.nf

‎song-score-download/local_modules/song-get-analysis.nf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../process/song_get_analysis.nf

‎song-score-download/main.nf‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
#!/usr/bin/env nextflow
2+
3+
/*
4+
Copyright (c) 2020-2021, Ontario Institute for Cancer Research
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
18+
Authors:
19+
Alex Lepsa
20+
Junjun Zhang
21+
*/
22+
23+
nextflow.enable.dsl = 2
24+
version = '2.6.0' // package version
25+
26+
// universal params go here, change default value as needed
27+
params.publish_dir = "" // set to empty string will disable publishDir
28+
29+
params.max_retries = 5 // set to 0 will disable retry
30+
params.first_retry_wait_time = 1 // in seconds
31+
32+
// tool specific parmas go here, add / change as needed
33+
params.study_id = "TEST-PR"
34+
params.analysis_id = "9940db0f-c100-496a-80db-0fc100d96ac1"
35+
36+
params.api_token = ""
37+
38+
params.song_cpus = 1
39+
params.song_mem = 1 // GB
40+
params.song_url = "https://song.rdpc-qa.cancercollaboratory.org"
41+
params.song_api_token = ""
42+
params.song_container_version = "4.2.1"
43+
44+
params.score_cpus = 1
45+
params.score_mem = 1 // GB
46+
params.score_transport_mem = 1 // GB
47+
params.score_url = "https://score.rdpc-qa.cancercollaboratory.org"
48+
params.score_api_token = ""
49+
params.score_container_version = "5.0.0"
50+
51+
52+
song_params = [
53+
*:params,
54+
'cpus': params.song_cpus,
55+
'mem': params.song_mem,
56+
'song_url': params.song_url,
57+
'song_container_version': params.song_container_version,
58+
'api_token': params.song_api_token ?: params.api_token,
59+
'publish_dir': ''
60+
]
61+
62+
score_params = [
63+
*:params,
64+
'cpus': params.score_cpus,
65+
'mem': params.score_mem,
66+
'transport_mem': params.score_transport_mem,
67+
'song_url': params.song_url,
68+
'score_url': params.score_url,
69+
'score_container_version': params.score_container_version,
70+
'api_token': params.score_api_token ?: params.api_token
71+
]
72+
73+
74+
include { songGetAnalysis } from './local_modules/song-get-analysis' params(song_params)
75+
include { scoreDownload } from './local_modules/score-download' params(score_params)
76+
77+
78+
// please update workflow code as needed
79+
workflow SongScoreDownload {
80+
take: // update as needed
81+
study_id
82+
analysis_id
83+
84+
main:
85+
songGetAnalysis(study_id, analysis_id)
86+
scoreDownload(songGetAnalysis.out.json, study_id, analysis_id)
87+
88+
emit:
89+
analysis_json = songGetAnalysis.out.json
90+
files = scoreDownload.out.files
91+
}
92+
93+
94+
// this provides an entry point for this main script, so it can be run directly without clone the repo
95+
// using this command: nextflow run <git_acc>/<repo>/<pkg_name>/<main_script>.nf -r <pkg_name>.v<pkg_version> --params-file xxx
96+
workflow {
97+
SongScoreDownload(
98+
params.study_id,
99+
params.analysis_id
100+
)
101+
}

‎song-score-download/nextflow.config‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
docker {
2+
enabled = true
3+
runOptions = '-u \$(id -u):\$(id -g)'
4+
}

‎song-score-download/pkg.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"name": "song-score-download",
3+
"version": "2.6.0",
4+
"description": "SONG/SCORE download",
5+
"main": "main.nf",
6+
"deprecated": false,
7+
"keywords": [
8+
"bioinformatics",
9+
"data",
10+
"metadata",
11+
"download"
12+
],
13+
"repository": {
14+
"type": "git",
15+
"url": "https://github.com/icgc-argo/nextflow-data-processing-utility-tools.git"
16+
},
17+
"dependencies": [],
18+
"devDependencies": [],
19+
"contributors": [
20+
{
21+
"name": "Alex Lepsa"
22+
},
23+
{
24+
"name": "Junjun Zhang"
25+
}
26+
],
27+
"license": "Apache License 2.0",
28+
"bugReport": "https://github.com/icgc-argo/nextflow-data-processing-utility-tools/issues",
29+
"homepage": "https://github.com/icgc-argo/nextflow-data-processing-utility-tools#readme"
30+
}

‎song-score-download/tests/checker.nf‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/usr/bin/env nextflow
2+
3+
/*
4+
Copyright (c) 2020-2021, Ontario Institute for Cancer Research
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
18+
Authors:
19+
Junjun Zhang
20+
*/
21+
22+
/*
23+
This is an auto-generated checker workflow to test the generated main template workflow, it's
24+
meant to illustrate how testing works. Please update to suit your own needs.
25+
*/
26+
27+
nextflow.enable.dsl = 2
28+
version = '2.6.0' // package version
29+
30+
// universal params
31+
params.publish_dir = ""
32+
33+
params.max_retries = 5 // set to 0 will disable retry
34+
params.first_retry_wait_time = 1 // in seconds
35+
36+
// tool specific parmas go here, add / change as needed
37+
params.study_id = ""
38+
params.analysis_id = ""
39+
40+
params.api_token = ""
41+
42+
params.song_cpus = 1
43+
params.song_mem = 1 // GB
44+
params.song_url = "https://song.rdpc-qa.cancercollaboratory.org"
45+
params.song_api_token = ""
46+
47+
params.score_cpus = 1
48+
params.score_mem = 1 // GB
49+
params.score_transport_mem = 1 // GB
50+
params.score_url = "https://score.rdpc-qa.cancercollaboratory.org"
51+
params.score_api_token = ""
52+
53+
54+
include { SongScoreDownload } from '../main'
55+
56+
57+
workflow checker {
58+
take:
59+
study_id
60+
analysis_id
61+
62+
main:
63+
SongScoreDownload(
64+
study_id,
65+
analysis_id
66+
)
67+
}
68+
69+
70+
workflow {
71+
checker(
72+
params.study_id,
73+
params.analysis_id
74+
)
75+
}

‎song-score-download/tests/local-test-1.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"study_id": "TEST-PR",
3+
"analysis_id": "9940db0f-c100-496a-80db-0fc100d96ac1",
4+
"publish_dir": "outdir"
5+
}

‎song-score-download/tests/local-test-2.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"study_id": "TEST-PR",
3+
"analysis_id": "9940db0f-c100-496a-80db-0fc100d96ac1",
4+
"max_retries": 3,
5+
"first_retry_wait_time": 5
6+
}

0 commit comments

Comments
 (0)