Skip to content

Commit acb3718

Browse files
junjun-zhangjunjun-zhang
committed
implemented cleanupWorkdir, added tests
1 parent 8aa4613 commit acb3718

13 files changed

Lines changed: 200 additions & 52 deletions

‎cleanup-workdir/Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ FROM ubuntu:20.04
22

33
LABEL org.opencontainers.image.source https://github.com/icgc-argo/data-processing-utility-tools
44

5-
ENV PATH="/tools:${PATH}"
6-
7-
COPY *.py /tools/
5+
RUN groupadd -g 1000 ubuntu && \
6+
useradd -l -u 1000 -g ubuntu ubuntu && \
7+
install -d -m 0755 -o ubuntu -g ubuntu /home/ubuntu
88

99
CMD ["/bin/bash"]

‎cleanup-workdir/tests/checker.nf‎

Lines changed: 55 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -46,52 +46,74 @@ params.container = ""
4646
params.input_file = ""
4747
params.expected_output = ""
4848

49+
params.cpus = 1
50+
params.mem = 1 // GB
51+
4952
include { cleanupWorkdir } from '../main'
5053

54+
include {
55+
generateDummyFile as gFile1;
56+
generateDummyFile as gFile2;
57+
} from './generate-dummy-file.nf'
5158

52-
process file_smart_diff {
53-
container "${params.container ?: container[params.container_registry ?: default_container_registry]}:${params.container_version ?: version}"
59+
include {
60+
filesExist as fExist1;
61+
filesExist as fExist2;
62+
filesExist as fExist3;
63+
filesExist as fExist4;
64+
} from './files-exist.nf'
5465

55-
input:
56-
path output_file
57-
path expected_file
66+
Channel.from(params.file_name).set{ file_name_ch }
5867

59-
output:
60-
stdout()
6168

62-
script:
63-
"""
64-
# Note: this is only for demo purpose, please write your own 'diff' according to your own needs.
65-
# remove date field before comparison eg, <div id="header_filename">Tue 19 Jan 2021<br/>test_rg_3.bam</div>
66-
# sed -e 's#"header_filename">.*<br/>test_rg_3.bam#"header_filename"><br/>test_rg_3.bam</div>#'
69+
workflow {
70+
// generate the file
71+
gFile1(
72+
file_name_ch.flatten(),
73+
params.file_size
74+
)
6775

68-
diff <( cat ${output_file} | sed -e 's#"header_filename">.*<br/>#"header_filename"><br/>#' ) \
69-
<( ([[ '${expected_file}' == *.gz ]] && gunzip -c ${expected_file} || cat ${expected_file}) | sed -e 's#"header_filename">.*<br/>#"header_filename"><br/>#' ) \
70-
&& ( echo "Test PASSED" && exit 0 ) || ( echo "Test FAILED, output file mismatch." && exit 1 )
71-
"""
72-
}
76+
// generate the same file again
77+
gFile2(
78+
file_name_ch.flatten(),
79+
params.file_size
80+
)
7381

82+
// test file exists
83+
fExist1(
84+
params.file_name,
85+
'exist',
86+
gFile1.out.file.collect(),
87+
true // no need to wait
88+
)
7489

75-
workflow checker {
76-
take:
77-
input_file
78-
expected_output
90+
// test file exist
91+
fExist2(
92+
params.file_name,
93+
'exist',
94+
gFile2.out.file.collect(),
95+
true // no need to wait
96+
)
7997

80-
main:
98+
// perform cleanup in gFile1 workdir
8199
cleanupWorkdir(
82-
input_file
100+
gFile1.out.collect(),
101+
gFile2.out.file.collect() // flag enables waiting for gFile2 before cleaning up gFile1 workdir
83102
)
84103

85-
file_smart_diff(
86-
cleanupWorkdir.out.output_file,
87-
expected_output
104+
// test cleaned up workdir from gFile1 indeed does not have previous files
105+
fExist3(
106+
gFile1.out.collect(),
107+
'not_exist',
108+
gFile1.out.collect(),
109+
cleanupWorkdir.out // wait for cleanup is done
88110
)
89-
}
90-
91111

92-
workflow {
93-
checker(
94-
file(params.input_file),
95-
file(params.expected_output)
96-
)
112+
// test not cleaned up workdir from gFile2 indeed still have the exptected files
113+
fExist4(
114+
gFile2.out.collect(),
115+
'exist',
116+
gFile2.out.collect(),
117+
cleanupWorkdir.out // wait for cleanup is done
118+
)
97119
}

‎cleanup-workdir/tests/expected/expected.test_rg_3.bam‎

-14.6 KB
Binary file not shown.

‎cleanup-workdir/tests/expected/expected.test_rg_3_fastqc.out.gz‎

-148 KB
Binary file not shown.

‎cleanup-workdir/tests/files-exist.nf‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env nextflow
2+
3+
/*
4+
* Copyright (c) 2019-2021, Ontario Institute for Cancer Research (OICR).
5+
*
6+
* This program is free software: you can redistribute it and/or modify
7+
* it under the terms of the GNU Affero General Public License as published
8+
* by the Free Software Foundation, either version 3 of the License, or
9+
* (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
* GNU Affero General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Affero General Public License
17+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
18+
*/
19+
20+
/*
21+
* Contributors:
22+
* Junjun Zhang <junjun.zhang@oicr.on.ca>
23+
*/
24+
25+
/********************************************************************/
26+
/* this block is auto-generated based on info from pkg.json where */
27+
/* changes can be made if needed, do NOT modify this block manually */
28+
nextflow.enable.dsl = 2
29+
version = '1.0.0' // package version
30+
31+
container = [
32+
'ghcr.io': 'ghcr.io/icgc-argo/data-processing-utility-tools.helper-functions'
33+
]
34+
default_container_registry = 'ghcr.io'
35+
/********************************************************************/
36+
37+
// universal params
38+
params.container_registry = ""
39+
params.container_version = ""
40+
params.container = ""
41+
42+
// generate dummy file to test cleanupWorkdir
43+
process filesExist {
44+
container "${params.container ?: container[params.container_registry ?: default_container_registry]}:${params.container_version ?: version}"
45+
46+
input:
47+
val file_names // file name shall not have spaces
48+
val expect // 'exist' for files expected to exist; 'not_exist' for files expected not exist
49+
path files
50+
val dependency_flag // any output from process(es) you'd like to make this process depend on
51+
52+
script:
53+
file_name_arg = file_names instanceof List ? file_names.join(" ") : file_names
54+
"""
55+
if [[ "${expect}" = "exist" ]]; then
56+
for f in \$(echo "${file_name_arg}"); do
57+
if [[ ! -f \$f ]]; then
58+
exit "Expected \$f not exists."
59+
fi
60+
done
61+
elif [[ "${expect}" = "not_exist" ]]; then
62+
for f in \$(echo "${file_name_arg}"); do
63+
if [[ -f \$f ]]; then
64+
exit "Unexpected \$f exists."
65+
fi
66+
done
67+
else
68+
exit "Second argument must be either 'exist' or 'not_exist'. '${expect}' is supplied."
69+
fi
70+
"""
71+
}

‎cleanup-workdir/tests/generate-dummy-file.nf‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/usr/bin/env nextflow
2+
3+
/*
4+
* Copyright (c) 2019-2021, Ontario Institute for Cancer Research (OICR).
5+
*
6+
* This program is free software: you can redistribute it and/or modify
7+
* it under the terms of the GNU Affero General Public License as published
8+
* by the Free Software Foundation, either version 3 of the License, or
9+
* (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
* GNU Affero General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Affero General Public License
17+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
18+
*/
19+
20+
/*
21+
* Contributors:
22+
* Junjun Zhang <junjun.zhang@oicr.on.ca>
23+
*/
24+
25+
/********************************************************************/
26+
/* this block is auto-generated based on info from pkg.json where */
27+
/* changes can be made if needed, do NOT modify this block manually */
28+
nextflow.enable.dsl = 2
29+
version = '1.0.0' // package version
30+
31+
container = [
32+
'ghcr.io': 'ghcr.io/icgc-argo/data-processing-utility-tools.helper-functions'
33+
]
34+
default_container_registry = 'ghcr.io'
35+
/********************************************************************/
36+
37+
// universal params
38+
params.container_registry = ""
39+
params.container_version = ""
40+
params.container = ""
41+
42+
process generateDummyFile {
43+
container "${params.container ?: container[params.container_registry ?: default_container_registry]}:${params.container_version ?: version}"
44+
45+
input:
46+
val file_name
47+
val file_size
48+
49+
output:
50+
path "*", emit: file
51+
52+
script:
53+
file_name_arg = file_name instanceof List ? file_name.join(".") : file_name
54+
"""
55+
dd if=/dev/urandom of="${file_name_arg}" bs=1 count=${file_size}
56+
"""
57+
}

‎cleanup-workdir/tests/input/README.md‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎cleanup-workdir/tests/input/test_rg_3.bam‎

-14.6 KB
Binary file not shown.

‎cleanup-workdir/tests/test-1.nf.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"file_name": "xyz.fa",
3+
"file_size": 1024
4+
}

‎cleanup-workdir/tests/test-2.nf.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"file_name": "abc.fa",
3+
"file_size": 512,
4+
"container_registry": "ghcr.io"
5+
}

0 commit comments

Comments
 (0)