From ebfd850a59c4bc62bd7118ae5aa1e137b3356004 Mon Sep 17 00:00:00 2001 From: jsabban <jules.sabban@inrae.fr> Date: Wed, 8 Jan 2025 09:47:47 +0100 Subject: [PATCH 1/9] Remove PlaGe related text in multiQC report ref: #143 --- assets/multiqc_config.yaml | 4 ---- lib/pipeline.groovy | 1 - 2 files changed, 5 deletions(-) diff --git a/assets/multiqc_config.yaml b/assets/multiqc_config.yaml index d311a87..c981be6 100644 --- a/assets/multiqc_config.yaml +++ b/assets/multiqc_config.yaml @@ -4,10 +4,6 @@ title: "My Title" #subtitle: "A subtitle to go underneath in grey" intro_text: "This MultiQC report summarise Quality Control analysis results." -report_comment: > - This report has been generated by the <a href="https://forgemia.inra.fr/get-nextflow-ngl-bi/wf-illumina-nf" target="_blank">wf-illumina-nf</a> - analysis pipeline. For information about how to interpret these results, please see the - <a href="https://forgemia.inra.fr/get-nextflow-ngl-bi/wf-illumina-nf" target="_blank">documentation</a>. show_analysis_paths: False show_analysis_time: False diff --git a/lib/pipeline.groovy b/lib/pipeline.groovy index 4cc505e..3690335 100644 --- a/lib/pipeline.groovy +++ b/lib/pipeline.groovy @@ -140,7 +140,6 @@ def getPipelineInfo() { def sn = params.machine_id ?: '' def platform = "$seq $sn" map['Sequencing Platform'] = platform ?: '' - map['Contact E-mail'] = 'get-plage.contact@genotoul.fr' return map } -- GitLab From 22339c124849a306de4b6770b468ba5ef149815e Mon Sep 17 00:00:00 2001 From: jsabban <jules.sabban@inrae.fr> Date: Thu, 16 Jan 2025 14:48:03 +0100 Subject: [PATCH 2/9] COPY_TO_SAVE wait NGSRG treatments completion Ref: #145 --- modules/local/module_NGL-Bi.nf | 2 ++ sub-workflows/local/ngl.nf | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/local/module_NGL-Bi.nf b/modules/local/module_NGL-Bi.nf index 59f2360..1e0aac7 100644 --- a/modules/local/module_NGL-Bi.nf +++ b/modules/local/module_NGL-Bi.nf @@ -82,6 +82,7 @@ process NGSRG_ILLUMINA { output: path("*.log"), emit: log + val 1, emit: ready script: def args = task.ext.args ?: '' @@ -107,6 +108,7 @@ process NGSRG_ELEMBIO { output: path("*.log"), emit: log + val 1, emit: ready script: def args = task.ext.args ?: '' diff --git a/sub-workflows/local/ngl.nf b/sub-workflows/local/ngl.nf index 34a1f01..21f7c02 100644 --- a/sub-workflows/local/ngl.nf +++ b/sub-workflows/local/ngl.nf @@ -48,10 +48,13 @@ workflow NGL { readsets_created = BEGIN.out.readsets_created ready = BEGIN.out.ready + ready_for_analysis = Channel.empty() if(sequencer_name =~ "NovaSeq|MiSeq") { NGSRG_ILLUMINA(readsets_created, demux_stat_json, 'readsets') + ready_for_analysis = NGSRG_ILLUMINA.out.ready } else if (sequencer_name =~ "AVITI") { NGSRG_ELEMBIO(readsets_created, params.inputdir, params.lane, 'readsets') + ready_for_analysis = NGSRG_ELEMBIO.out.ready } bi_run_code = nglBiRunCode.collect().map { it.toString() } @@ -80,7 +83,8 @@ workflow NGL { project_hash, fq, MD5SUM_FASTQ.out.md5sum, - params.run_name + params.run_name, + ready_for_analysis ) emit: -- GitLab From 8db93fb7f457bd52d1748a2917e378453371a284 Mon Sep 17 00:00:00 2001 From: jsabban <jules.sabban@inrae.fr> Date: Thu, 16 Jan 2025 15:51:38 +0100 Subject: [PATCH 3/9] Use sample list for readset creation Ref: #144 --- conf/base.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/base.config b/conf/base.config index 4bc6650..0a0bffd 100644 --- a/conf/base.config +++ b/conf/base.config @@ -262,6 +262,7 @@ process { withName: CREATE_READSETS { ext.args = [ params.run_name ? "--poolName '${params.run_name}'" : '', + params.select_samples ? "--samples '${params.select_samples}'" : '', params.lane ? "--laneNumberToWorkOn '${params.lane}'" : '', ].join(' ') } -- GitLab From e35d9d0734fbde1ac577ca501ccdb14ab8e8d4e0 Mon Sep 17 00:00:00 2001 From: jsabban <jules.sabban@inrae.fr> Date: Tue, 21 Jan 2025 17:50:41 +0100 Subject: [PATCH 4/9] Group fastq and md5 together To fit with changes in shared_modules/copy_to_save Ref: #147 --- sub-workflows/local/ngl.nf | 32 ++++++++++++++++++++++++-------- workflow/short_reads_qc.nf | 2 ++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/sub-workflows/local/ngl.nf b/sub-workflows/local/ngl.nf index 21f7c02..29c12bb 100644 --- a/sub-workflows/local/ngl.nf +++ b/sub-workflows/local/ngl.nf @@ -14,10 +14,9 @@ include { BEGIN_NGLBI as BEGIN } from "${params.shared_modules}/workflows/ include { COPY_TO_SAVE } from "${params.shared_modules}/workflows/copy_to_save.nf" include { MD5SUM as MD5SUM_FASTQ; md5sum as MD5SUM_INDEX } from "${params.shared_modules}/md5sum.nf" -include { READSET_FILE_FROM_FILE as ADD_RS_INDEX_FILES } from "${params.shared_modules}/ngl_bi.nf" addParams(ext: 'INDEX') -include { READSET_FILE_FROM_FILE as ADD_RS_RAW_FILES } from "${params.shared_modules}/ngl_bi.nf" addParams(ext: 'RAW') -include { UPDATE_NGLBI_STATE_FROM_FILE as UPDATE_STATE_FQC; - CREATE_ANALYSIS } from "${params.shared_modules}/ngl_bi.nf" +include { READSET_FILE_FROM_FILE as ADD_RS_INDEX_FILES } from "${params.shared_modules}/ngl_bi.nf" +include { READSET_FILE_FROM_FILE as ADD_RS_RAW_FILES } from "${params.shared_modules}/ngl_bi.nf" +include { UPDATE_NGLBI_STATE_FROM_FILE as UPDATE_STATE_FQC} from "${params.shared_modules}/ngl_bi.nf" include { FILE_RENAME as RENAME_FASTQ; FILE_RENAME as RENAME_INDEX; @@ -41,8 +40,11 @@ workflow NGL { demux_stat_json main: + fastq_files = Channel.empty() + md5_files = Channel.empty() + // First steps - BEGIN(bi_run_code, sq_xp_code, '', sequencer_name) + BEGIN(bi_run_code, sq_xp_code,'', sequencer_name) nglBiRunCode = BEGIN.out.nglBiRunCode readsets_created = BEGIN.out.readsets_created @@ -62,26 +64,40 @@ workflow NGL { // Compute results if (is_single_cell) { RENAME_INDEX(index_files.map{it[1]}.collect(), readsets_created, sq_xp_code, 'fastq_index') - MD5SUM_INDEX(RENAME_INDEX.out.fastq.collect(), run_name+'_fastq_index') + MD5SUM_INDEX(RENAME_INDEX.out.fastq.collect(), 'fastq_index_'+run_name) ADD_RS_INDEX_FILES(readsets_created, MD5SUM_INDEX.out, 'INDEX', ready) + + fastq_files = fastq_files.mix(RENAME_INDEX.out.fastq) + md5_files = md5_files.mix(MD5SUM_INDEX.out + .map { it -> [[type: 'fastq', barcode:run_name], it]} + ) } RENAME_FASTQ(fastq.map{it[1]}.collect(), readsets_created, sq_xp_code, 'fastq_read') - fq = RENAME_FASTQ.out.fastq + fastq_files = fastq_files.mix(RENAME_FASTQ.out.fastq.ifEmpty([])) + + fq = fastq_files .flatMap() .map { it -> [[type: 'fastq', barcode:run_name], it]} MD5SUM_FASTQ(fq.groupTuple()) + md5_files = md5_files.mix(MD5SUM_FASTQ.out.md5sum.ifEmpty([])).view() UPDATE_STATE_FQC(readsets_created, 'F-QC', multiqc_html) techno = sequencer_name =~ "MiSeq|NovaSeq" ? 'illumina' : sequencer_name =~ "AVITI" ? 'elembio' : 'other' + + fq_md5 = fq + .map{it -> [[type:'fastq', barcode:'all'], it[1]]} + .mix(md5_files.map{it -> [[type:'fastq', barcode:'all'], it[1]]}) + .groupTuple() + COPY_TO_SAVE( nglBiRunCode, readsets_created, multiqc_html.map{it -> [[type:'report'], it]}, project, project_hash, - fq, + fq_md5, MD5SUM_FASTQ.out.md5sum, params.run_name, ready_for_analysis diff --git a/workflow/short_reads_qc.nf b/workflow/short_reads_qc.nf index 5dd432a..f02ca31 100644 --- a/workflow/short_reads_qc.nf +++ b/workflow/short_reads_qc.nf @@ -119,9 +119,11 @@ workflow SHORT_READS_QC { if (params.skip_core_element) { log.info "Elembio's sequencer but skipping CORE_ELEMENT" demux_stats = Channel.empty() + fastq = ch_read } else { CORE_ELEMENT(ch_runManifestJSON, ch_indexAssigned, ch_indexUnassigned) demux_stats = CORE_ELEMENT.out.demuxStat + fastq = ch_read } } -- GitLab From 5d98d07504c04ea600266e2456b56c9635af51e9 Mon Sep 17 00:00:00 2001 From: jsabban <jules.sabban@inrae.fr> Date: Tue, 21 Jan 2025 18:03:19 +0100 Subject: [PATCH 5/9] Deals with new elembio config file and env Ref: #148 --- conf/prod.config | 3 ++- conf/test.config | 8 ++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/conf/prod.config b/conf/prod.config index eb1bd1a..0670687 100644 --- a/conf/prod.config +++ b/conf/prod.config @@ -3,7 +3,8 @@ //========================================= process { withLabel: ngl { - beforeScript = "source ${params.ngl_bi_client}/GeT/bash/loadConfFile.sh ${params.ngl_bi_client}/IG/SystemeInteractionNGL-Bi/conf/prod_illumina_qc.conf" + def techno_label = params.sequencer == 'AVITI' ? 'elembio' : 'illumina' + beforeScript = "source ${params.ngl_bi_client}/GeT/bash/loadConfFile.sh ${params.ngl_bi_client}/IG/SystemeInteractionNGL-Bi/conf/prod_${techno_label}_qc.conf" publishDir = [ path: "${params.outdir}/ngl", mode: 'copy', diff --git a/conf/test.config b/conf/test.config index c2de5cd..a795043 100644 --- a/conf/test.config +++ b/conf/test.config @@ -10,7 +10,8 @@ params { process { withLabel: ngl { - beforeScript = "source ${params.ngl_bi_client}/GeT/bash/loadConfFile.sh ${params.ngl_bi_client}/IG/SystemeInteractionNGL-Bi/conf/dev_illumina_qc.conf" + def techno_label = params.sequencer == 'AVITI' ? 'elembio' : 'illumina' + beforeScript = "source ${params.ngl_bi_client}/GeT/bash/loadConfFile.sh ${params.ngl_bi_client}/IG/SystemeInteractionNGL-Bi/conf/dev_${techno_label}_qc.conf" publishDir = [ path: "${params.outdir}/ngl", mode: 'copy', @@ -18,9 +19,4 @@ process { ] } - withName: CREATE_READSETS { - ext.args = [ - params.lane ? "--laneNumberToWorkOn '${params.lane}'" : '', - ].join(' ') - } } -- GitLab From 04a8475af3fc8625bc043af0c3931512e32b7545 Mon Sep 17 00:00:00 2001 From: jsabban <jules.sabban@inrae.fr> Date: Tue, 21 Jan 2025 18:15:42 +0100 Subject: [PATCH 6/9] Ignore some error in analysis creation and more time to store fastq files into /save/ngl Ref: #149 --- conf/base.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/base.config b/conf/base.config index 0a0bffd..0be9913 100644 --- a/conf/base.config +++ b/conf/base.config @@ -255,8 +255,13 @@ process { // SHARED MODULES //========================================= process { + withName: STORE_FASTQ { + time = { checkMax( 3.h * task.attempt, 'time' ) } + } + withName: CREATE_ANALYSIS { ext.args = params.project_hash ? "--projectDir '${params.project}-${params.project_hash}'" : '' + errorStrategy = { task.exitStatus in [6] ? 'ignore' : 'retry' } } withName: CREATE_READSETS { -- GitLab From e56bc9b3391bcfada2be4fb77494cebd0c0f44ee Mon Sep 17 00:00:00 2001 From: jsabban <jules.sabban@inrae.fr> Date: Thu, 23 Jan 2025 16:58:51 +0100 Subject: [PATCH 7/9] Remove unwantes graphs in multiQC config Ref: #150 --- assets/multiqc_config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/assets/multiqc_config.yaml b/assets/multiqc_config.yaml index c981be6..c83e60d 100644 --- a/assets/multiqc_config.yaml +++ b/assets/multiqc_config.yaml @@ -49,6 +49,12 @@ report_section_order: summary: order: -1001 +remove_sections: + - fastp-seq-content-n + - fastp-seq-quality + - fastp-seq-content-gc + - flash-histogram + module_order: - fastqc: name: "ReadsStats" -- GitLab From 4effb395fbe33a02a6adf2c659c03cebba11573f Mon Sep 17 00:00:00 2001 From: jsabban <jules.sabban@inrae.fr> Date: Thu, 23 Jan 2025 17:04:41 +0100 Subject: [PATCH 8/9] Remove from ngl subworkflow md5sum and readset Files creations Ref: #151 --- sub-workflows/local/ngl.nf | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/sub-workflows/local/ngl.nf b/sub-workflows/local/ngl.nf index 29c12bb..f05c720 100644 --- a/sub-workflows/local/ngl.nf +++ b/sub-workflows/local/ngl.nf @@ -12,16 +12,12 @@ // ------------------------------------------------- include { BEGIN_NGLBI as BEGIN } from "${params.shared_modules}/workflows/begin_nglbi.nf" include { COPY_TO_SAVE } from "${params.shared_modules}/workflows/copy_to_save.nf" -include { MD5SUM as MD5SUM_FASTQ; - md5sum as MD5SUM_INDEX } from "${params.shared_modules}/md5sum.nf" -include { READSET_FILE_FROM_FILE as ADD_RS_INDEX_FILES } from "${params.shared_modules}/ngl_bi.nf" -include { READSET_FILE_FROM_FILE as ADD_RS_RAW_FILES } from "${params.shared_modules}/ngl_bi.nf" include { UPDATE_NGLBI_STATE_FROM_FILE as UPDATE_STATE_FQC} from "${params.shared_modules}/ngl_bi.nf" -include { FILE_RENAME as RENAME_FASTQ; - FILE_RENAME as RENAME_INDEX; - NGSRG_ILLUMINA; - NGSRG_ELEMBIO; } from "$baseDir/modules/local/module_NGL-Bi.nf" +include { FILE_RENAME as RENAME_FASTQ } from "$baseDir/modules/local/module_NGL-Bi.nf" +include { FILE_RENAME as RENAME_INDEX } from "$baseDir/modules/local/module_NGL-Bi.nf" +include { NGSRG_ILLUMINA } from "$baseDir/modules/local/module_NGL-Bi.nf" +include { NGSRG_ELEMBIO; } from "$baseDir/modules/local/module_NGL-Bi.nf" // ------------------------------------------------- // WORKFLOW // ------------------------------------------------- @@ -64,13 +60,7 @@ workflow NGL { // Compute results if (is_single_cell) { RENAME_INDEX(index_files.map{it[1]}.collect(), readsets_created, sq_xp_code, 'fastq_index') - MD5SUM_INDEX(RENAME_INDEX.out.fastq.collect(), 'fastq_index_'+run_name) - ADD_RS_INDEX_FILES(readsets_created, MD5SUM_INDEX.out, 'INDEX', ready) - fastq_files = fastq_files.mix(RENAME_INDEX.out.fastq) - md5_files = md5_files.mix(MD5SUM_INDEX.out - .map { it -> [[type: 'fastq', barcode:run_name], it]} - ) } RENAME_FASTQ(fastq.map{it[1]}.collect(), readsets_created, sq_xp_code, 'fastq_read') @@ -78,18 +68,10 @@ workflow NGL { fq = fastq_files .flatMap() - .map { it -> [[type: 'fastq', barcode:run_name], it]} - - MD5SUM_FASTQ(fq.groupTuple()) - md5_files = md5_files.mix(MD5SUM_FASTQ.out.md5sum.ifEmpty([])).view() + .map { it -> [[type: 'fastq', barcode:'all'], it]} + .groupTuple() UPDATE_STATE_FQC(readsets_created, 'F-QC', multiqc_html) - techno = sequencer_name =~ "MiSeq|NovaSeq" ? 'illumina' : sequencer_name =~ "AVITI" ? 'elembio' : 'other' - - fq_md5 = fq - .map{it -> [[type:'fastq', barcode:'all'], it[1]]} - .mix(md5_files.map{it -> [[type:'fastq', barcode:'all'], it[1]]}) - .groupTuple() COPY_TO_SAVE( nglBiRunCode, @@ -97,8 +79,7 @@ workflow NGL { multiqc_html.map{it -> [[type:'report'], it]}, project, project_hash, - fq_md5, - MD5SUM_FASTQ.out.md5sum, + fq, params.run_name, ready_for_analysis ) -- GitLab From e9fed7ababa4f22b1d5fe5683b7d430efdf13e09 Mon Sep 17 00:00:00 2001 From: jsabban <jules.sabban@inrae.fr> Date: Mon, 27 Jan 2025 09:33:42 +0100 Subject: [PATCH 9/9] Update manifest version --- conf/report.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/report.config b/conf/report.config index 42aef54..7cd6657 100644 --- a/conf/report.config +++ b/conf/report.config @@ -29,5 +29,5 @@ manifest { description = "Workflow for Illumina data quality control" mainScript = 'main.nf' nextflowVersion = '>=0.32.0' - version = '1.26.5' + version = '1.27.0' } \ No newline at end of file -- GitLab