From ebfd850a59c4bc62bd7118ae5aa1e137b3356004 Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Wed, 8 Jan 2025 09:47:47 +0100
Subject: [PATCH 1/9] Remove PlaGe related text in multiQC report
ref: #143
---
assets/multiqc_config.yaml | 4 ----
lib/pipeline.groovy | 1 -
2 files changed, 5 deletions(-)
diff --git a/assets/multiqc_config.yaml b/assets/multiqc_config.yaml
index d311a87..c981be6 100644
--- a/assets/multiqc_config.yaml
+++ b/assets/multiqc_config.yaml
@@ -4,10 +4,6 @@ title: "My Title"
#subtitle: "A subtitle to go underneath in grey"
intro_text: "This MultiQC report summarise Quality Control analysis results."
-report_comment: >
- This report has been generated by the <a href="https://forgemia.inra.fr/get-nextflow-ngl-bi/wf-illumina-nf" target="_blank">wf-illumina-nf</a>
- analysis pipeline. For information about how to interpret these results, please see the
- <a href="https://forgemia.inra.fr/get-nextflow-ngl-bi/wf-illumina-nf" target="_blank">documentation</a>.
show_analysis_paths: False
show_analysis_time: False
diff --git a/lib/pipeline.groovy b/lib/pipeline.groovy
index 4cc505e..3690335 100644
--- a/lib/pipeline.groovy
+++ b/lib/pipeline.groovy
@@ -140,7 +140,6 @@ def getPipelineInfo() {
def sn = params.machine_id ?: ''
def platform = "$seq $sn"
map['Sequencing Platform'] = platform ?: ''
- map['Contact E-mail'] = 'get-plage.contact@genotoul.fr'
return map
}
--
GitLab
From 22339c124849a306de4b6770b468ba5ef149815e Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Thu, 16 Jan 2025 14:48:03 +0100
Subject: [PATCH 2/9] COPY_TO_SAVE wait NGSRG treatments completion
Ref: #145
---
modules/local/module_NGL-Bi.nf | 2 ++
sub-workflows/local/ngl.nf | 6 +++++-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/modules/local/module_NGL-Bi.nf b/modules/local/module_NGL-Bi.nf
index 59f2360..1e0aac7 100644
--- a/modules/local/module_NGL-Bi.nf
+++ b/modules/local/module_NGL-Bi.nf
@@ -82,6 +82,7 @@ process NGSRG_ILLUMINA {
output:
path("*.log"), emit: log
+ val 1, emit: ready
script:
def args = task.ext.args ?: ''
@@ -107,6 +108,7 @@ process NGSRG_ELEMBIO {
output:
path("*.log"), emit: log
+ val 1, emit: ready
script:
def args = task.ext.args ?: ''
diff --git a/sub-workflows/local/ngl.nf b/sub-workflows/local/ngl.nf
index 34a1f01..21f7c02 100644
--- a/sub-workflows/local/ngl.nf
+++ b/sub-workflows/local/ngl.nf
@@ -48,10 +48,13 @@ workflow NGL {
readsets_created = BEGIN.out.readsets_created
ready = BEGIN.out.ready
+ ready_for_analysis = Channel.empty()
if(sequencer_name =~ "NovaSeq|MiSeq") {
NGSRG_ILLUMINA(readsets_created, demux_stat_json, 'readsets')
+ ready_for_analysis = NGSRG_ILLUMINA.out.ready
} else if (sequencer_name =~ "AVITI") {
NGSRG_ELEMBIO(readsets_created, params.inputdir, params.lane, 'readsets')
+ ready_for_analysis = NGSRG_ELEMBIO.out.ready
}
bi_run_code = nglBiRunCode.collect().map { it.toString() }
@@ -80,7 +83,8 @@ workflow NGL {
project_hash,
fq,
MD5SUM_FASTQ.out.md5sum,
- params.run_name
+ params.run_name,
+ ready_for_analysis
)
emit:
--
GitLab
From 8db93fb7f457bd52d1748a2917e378453371a284 Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Thu, 16 Jan 2025 15:51:38 +0100
Subject: [PATCH 3/9] Use sample list for readset creation
Ref: #144
---
conf/base.config | 1 +
1 file changed, 1 insertion(+)
diff --git a/conf/base.config b/conf/base.config
index 4bc6650..0a0bffd 100644
--- a/conf/base.config
+++ b/conf/base.config
@@ -262,6 +262,7 @@ process {
withName: CREATE_READSETS {
ext.args = [
params.run_name ? "--poolName '${params.run_name}'" : '',
+ params.select_samples ? "--samples '${params.select_samples}'" : '',
params.lane ? "--laneNumberToWorkOn '${params.lane}'" : '',
].join(' ')
}
--
GitLab
From e35d9d0734fbde1ac577ca501ccdb14ab8e8d4e0 Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Tue, 21 Jan 2025 17:50:41 +0100
Subject: [PATCH 4/9] Group fastq and md5 together
To fit with changes in shared_modules/copy_to_save
Ref: #147
---
sub-workflows/local/ngl.nf | 32 ++++++++++++++++++++++++--------
workflow/short_reads_qc.nf | 2 ++
2 files changed, 26 insertions(+), 8 deletions(-)
diff --git a/sub-workflows/local/ngl.nf b/sub-workflows/local/ngl.nf
index 21f7c02..29c12bb 100644
--- a/sub-workflows/local/ngl.nf
+++ b/sub-workflows/local/ngl.nf
@@ -14,10 +14,9 @@ include { BEGIN_NGLBI as BEGIN } from "${params.shared_modules}/workflows/
include { COPY_TO_SAVE } from "${params.shared_modules}/workflows/copy_to_save.nf"
include { MD5SUM as MD5SUM_FASTQ;
md5sum as MD5SUM_INDEX } from "${params.shared_modules}/md5sum.nf"
-include { READSET_FILE_FROM_FILE as ADD_RS_INDEX_FILES } from "${params.shared_modules}/ngl_bi.nf" addParams(ext: 'INDEX')
-include { READSET_FILE_FROM_FILE as ADD_RS_RAW_FILES } from "${params.shared_modules}/ngl_bi.nf" addParams(ext: 'RAW')
-include { UPDATE_NGLBI_STATE_FROM_FILE as UPDATE_STATE_FQC;
- CREATE_ANALYSIS } from "${params.shared_modules}/ngl_bi.nf"
+include { READSET_FILE_FROM_FILE as ADD_RS_INDEX_FILES } from "${params.shared_modules}/ngl_bi.nf"
+include { READSET_FILE_FROM_FILE as ADD_RS_RAW_FILES } from "${params.shared_modules}/ngl_bi.nf"
+include { UPDATE_NGLBI_STATE_FROM_FILE as UPDATE_STATE_FQC} from "${params.shared_modules}/ngl_bi.nf"
include { FILE_RENAME as RENAME_FASTQ;
FILE_RENAME as RENAME_INDEX;
@@ -41,8 +40,11 @@ workflow NGL {
demux_stat_json
main:
+ fastq_files = Channel.empty()
+ md5_files = Channel.empty()
+
// First steps
- BEGIN(bi_run_code, sq_xp_code, '', sequencer_name)
+ BEGIN(bi_run_code, sq_xp_code,'', sequencer_name)
nglBiRunCode = BEGIN.out.nglBiRunCode
readsets_created = BEGIN.out.readsets_created
@@ -62,26 +64,40 @@ workflow NGL {
// Compute results
if (is_single_cell) {
RENAME_INDEX(index_files.map{it[1]}.collect(), readsets_created, sq_xp_code, 'fastq_index')
- MD5SUM_INDEX(RENAME_INDEX.out.fastq.collect(), run_name+'_fastq_index')
+ MD5SUM_INDEX(RENAME_INDEX.out.fastq.collect(), 'fastq_index_'+run_name)
ADD_RS_INDEX_FILES(readsets_created, MD5SUM_INDEX.out, 'INDEX', ready)
+
+ fastq_files = fastq_files.mix(RENAME_INDEX.out.fastq)
+ md5_files = md5_files.mix(MD5SUM_INDEX.out
+ .map { it -> [[type: 'fastq', barcode:run_name], it]}
+ )
}
RENAME_FASTQ(fastq.map{it[1]}.collect(), readsets_created, sq_xp_code, 'fastq_read')
- fq = RENAME_FASTQ.out.fastq
+ fastq_files = fastq_files.mix(RENAME_FASTQ.out.fastq.ifEmpty([]))
+
+ fq = fastq_files
.flatMap()
.map { it -> [[type: 'fastq', barcode:run_name], it]}
MD5SUM_FASTQ(fq.groupTuple())
+ md5_files = md5_files.mix(MD5SUM_FASTQ.out.md5sum.ifEmpty([])).view()
UPDATE_STATE_FQC(readsets_created, 'F-QC', multiqc_html)
techno = sequencer_name =~ "MiSeq|NovaSeq" ? 'illumina' : sequencer_name =~ "AVITI" ? 'elembio' : 'other'
+
+ fq_md5 = fq
+ .map{it -> [[type:'fastq', barcode:'all'], it[1]]}
+ .mix(md5_files.map{it -> [[type:'fastq', barcode:'all'], it[1]]})
+ .groupTuple()
+
COPY_TO_SAVE(
nglBiRunCode,
readsets_created,
multiqc_html.map{it -> [[type:'report'], it]},
project,
project_hash,
- fq,
+ fq_md5,
MD5SUM_FASTQ.out.md5sum,
params.run_name,
ready_for_analysis
diff --git a/workflow/short_reads_qc.nf b/workflow/short_reads_qc.nf
index 5dd432a..f02ca31 100644
--- a/workflow/short_reads_qc.nf
+++ b/workflow/short_reads_qc.nf
@@ -119,9 +119,11 @@ workflow SHORT_READS_QC {
if (params.skip_core_element) {
log.info "Elembio's sequencer but skipping CORE_ELEMENT"
demux_stats = Channel.empty()
+ fastq = ch_read
} else {
CORE_ELEMENT(ch_runManifestJSON, ch_indexAssigned, ch_indexUnassigned)
demux_stats = CORE_ELEMENT.out.demuxStat
+ fastq = ch_read
}
}
--
GitLab
From 5d98d07504c04ea600266e2456b56c9635af51e9 Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Tue, 21 Jan 2025 18:03:19 +0100
Subject: [PATCH 5/9] Deals with new elembio config file and env
Ref: #148
---
conf/prod.config | 3 ++-
conf/test.config | 8 ++------
2 files changed, 4 insertions(+), 7 deletions(-)
diff --git a/conf/prod.config b/conf/prod.config
index eb1bd1a..0670687 100644
--- a/conf/prod.config
+++ b/conf/prod.config
@@ -3,7 +3,8 @@
//=========================================
process {
withLabel: ngl {
- beforeScript = "source ${params.ngl_bi_client}/GeT/bash/loadConfFile.sh ${params.ngl_bi_client}/IG/SystemeInteractionNGL-Bi/conf/prod_illumina_qc.conf"
+ def techno_label = params.sequencer == 'AVITI' ? 'elembio' : 'illumina'
+ beforeScript = "source ${params.ngl_bi_client}/GeT/bash/loadConfFile.sh ${params.ngl_bi_client}/IG/SystemeInteractionNGL-Bi/conf/prod_${techno_label}_qc.conf"
publishDir = [
path: "${params.outdir}/ngl",
mode: 'copy',
diff --git a/conf/test.config b/conf/test.config
index c2de5cd..a795043 100644
--- a/conf/test.config
+++ b/conf/test.config
@@ -10,7 +10,8 @@ params {
process {
withLabel: ngl {
- beforeScript = "source ${params.ngl_bi_client}/GeT/bash/loadConfFile.sh ${params.ngl_bi_client}/IG/SystemeInteractionNGL-Bi/conf/dev_illumina_qc.conf"
+ def techno_label = params.sequencer == 'AVITI' ? 'elembio' : 'illumina'
+ beforeScript = "source ${params.ngl_bi_client}/GeT/bash/loadConfFile.sh ${params.ngl_bi_client}/IG/SystemeInteractionNGL-Bi/conf/dev_${techno_label}_qc.conf"
publishDir = [
path: "${params.outdir}/ngl",
mode: 'copy',
@@ -18,9 +19,4 @@ process {
]
}
- withName: CREATE_READSETS {
- ext.args = [
- params.lane ? "--laneNumberToWorkOn '${params.lane}'" : '',
- ].join(' ')
- }
}
--
GitLab
From 04a8475af3fc8625bc043af0c3931512e32b7545 Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Tue, 21 Jan 2025 18:15:42 +0100
Subject: [PATCH 6/9] Ignore some error in analysis creation
and more time to store fastq files into /save/ngl
Ref: #149
---
conf/base.config | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/conf/base.config b/conf/base.config
index 0a0bffd..0be9913 100644
--- a/conf/base.config
+++ b/conf/base.config
@@ -255,8 +255,13 @@ process {
// SHARED MODULES
//=========================================
process {
+ withName: STORE_FASTQ {
+ time = { checkMax( 3.h * task.attempt, 'time' ) }
+ }
+
withName: CREATE_ANALYSIS {
ext.args = params.project_hash ? "--projectDir '${params.project}-${params.project_hash}'" : ''
+ errorStrategy = { task.exitStatus in [6] ? 'ignore' : 'retry' }
}
withName: CREATE_READSETS {
--
GitLab
From e56bc9b3391bcfada2be4fb77494cebd0c0f44ee Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Thu, 23 Jan 2025 16:58:51 +0100
Subject: [PATCH 7/9] Remove unwantes graphs in multiQC config
Ref: #150
---
assets/multiqc_config.yaml | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/assets/multiqc_config.yaml b/assets/multiqc_config.yaml
index c981be6..c83e60d 100644
--- a/assets/multiqc_config.yaml
+++ b/assets/multiqc_config.yaml
@@ -49,6 +49,12 @@ report_section_order:
summary:
order: -1001
+remove_sections:
+ - fastp-seq-content-n
+ - fastp-seq-quality
+ - fastp-seq-content-gc
+ - flash-histogram
+
module_order:
- fastqc:
name: "ReadsStats"
--
GitLab
From 4effb395fbe33a02a6adf2c659c03cebba11573f Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Thu, 23 Jan 2025 17:04:41 +0100
Subject: [PATCH 8/9] Remove from ngl subworkflow md5sum and readset Files
creations
Ref: #151
---
sub-workflows/local/ngl.nf | 33 +++++++--------------------------
1 file changed, 7 insertions(+), 26 deletions(-)
diff --git a/sub-workflows/local/ngl.nf b/sub-workflows/local/ngl.nf
index 29c12bb..f05c720 100644
--- a/sub-workflows/local/ngl.nf
+++ b/sub-workflows/local/ngl.nf
@@ -12,16 +12,12 @@
// -------------------------------------------------
include { BEGIN_NGLBI as BEGIN } from "${params.shared_modules}/workflows/begin_nglbi.nf"
include { COPY_TO_SAVE } from "${params.shared_modules}/workflows/copy_to_save.nf"
-include { MD5SUM as MD5SUM_FASTQ;
- md5sum as MD5SUM_INDEX } from "${params.shared_modules}/md5sum.nf"
-include { READSET_FILE_FROM_FILE as ADD_RS_INDEX_FILES } from "${params.shared_modules}/ngl_bi.nf"
-include { READSET_FILE_FROM_FILE as ADD_RS_RAW_FILES } from "${params.shared_modules}/ngl_bi.nf"
include { UPDATE_NGLBI_STATE_FROM_FILE as UPDATE_STATE_FQC} from "${params.shared_modules}/ngl_bi.nf"
-include { FILE_RENAME as RENAME_FASTQ;
- FILE_RENAME as RENAME_INDEX;
- NGSRG_ILLUMINA;
- NGSRG_ELEMBIO; } from "$baseDir/modules/local/module_NGL-Bi.nf"
+include { FILE_RENAME as RENAME_FASTQ } from "$baseDir/modules/local/module_NGL-Bi.nf"
+include { FILE_RENAME as RENAME_INDEX } from "$baseDir/modules/local/module_NGL-Bi.nf"
+include { NGSRG_ILLUMINA } from "$baseDir/modules/local/module_NGL-Bi.nf"
+include { NGSRG_ELEMBIO; } from "$baseDir/modules/local/module_NGL-Bi.nf"
// -------------------------------------------------
// WORKFLOW
// -------------------------------------------------
@@ -64,13 +60,7 @@ workflow NGL {
// Compute results
if (is_single_cell) {
RENAME_INDEX(index_files.map{it[1]}.collect(), readsets_created, sq_xp_code, 'fastq_index')
- MD5SUM_INDEX(RENAME_INDEX.out.fastq.collect(), 'fastq_index_'+run_name)
- ADD_RS_INDEX_FILES(readsets_created, MD5SUM_INDEX.out, 'INDEX', ready)
-
fastq_files = fastq_files.mix(RENAME_INDEX.out.fastq)
- md5_files = md5_files.mix(MD5SUM_INDEX.out
- .map { it -> [[type: 'fastq', barcode:run_name], it]}
- )
}
RENAME_FASTQ(fastq.map{it[1]}.collect(), readsets_created, sq_xp_code, 'fastq_read')
@@ -78,18 +68,10 @@ workflow NGL {
fq = fastq_files
.flatMap()
- .map { it -> [[type: 'fastq', barcode:run_name], it]}
-
- MD5SUM_FASTQ(fq.groupTuple())
- md5_files = md5_files.mix(MD5SUM_FASTQ.out.md5sum.ifEmpty([])).view()
+ .map { it -> [[type: 'fastq', barcode:'all'], it]}
+ .groupTuple()
UPDATE_STATE_FQC(readsets_created, 'F-QC', multiqc_html)
- techno = sequencer_name =~ "MiSeq|NovaSeq" ? 'illumina' : sequencer_name =~ "AVITI" ? 'elembio' : 'other'
-
- fq_md5 = fq
- .map{it -> [[type:'fastq', barcode:'all'], it[1]]}
- .mix(md5_files.map{it -> [[type:'fastq', barcode:'all'], it[1]]})
- .groupTuple()
COPY_TO_SAVE(
nglBiRunCode,
@@ -97,8 +79,7 @@ workflow NGL {
multiqc_html.map{it -> [[type:'report'], it]},
project,
project_hash,
- fq_md5,
- MD5SUM_FASTQ.out.md5sum,
+ fq,
params.run_name,
ready_for_analysis
)
--
GitLab
From e9fed7ababa4f22b1d5fe5683b7d430efdf13e09 Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Mon, 27 Jan 2025 09:33:42 +0100
Subject: [PATCH 9/9] Update manifest version
---
conf/report.config | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/conf/report.config b/conf/report.config
index 42aef54..7cd6657 100644
--- a/conf/report.config
+++ b/conf/report.config
@@ -29,5 +29,5 @@ manifest {
description = "Workflow for Illumina data quality control"
mainScript = 'main.nf'
nextflowVersion = '>=0.32.0'
- version = '1.26.5'
+ version = '1.27.0'
}
\ No newline at end of file
--
GitLab