From ebfd850a59c4bc62bd7118ae5aa1e137b3356004 Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Wed, 8 Jan 2025 09:47:47 +0100
Subject: [PATCH 1/9] Remove PlaGe related text in multiQC report

	ref: #143
---
 assets/multiqc_config.yaml | 4 ----
 lib/pipeline.groovy        | 1 -
 2 files changed, 5 deletions(-)

diff --git a/assets/multiqc_config.yaml b/assets/multiqc_config.yaml
index d311a87..c981be6 100644
--- a/assets/multiqc_config.yaml
+++ b/assets/multiqc_config.yaml
@@ -4,10 +4,6 @@ title: "My Title"
 #subtitle: "A subtitle to go underneath in grey"
 intro_text: "This MultiQC report summarise Quality Control analysis results."
 
-report_comment: >
-    This report has been generated by the <a href="https://forgemia.inra.fr/get-nextflow-ngl-bi/wf-illumina-nf" target="_blank">wf-illumina-nf</a>
-    analysis pipeline. For information about how to interpret these results, please see the
-    <a href="https://forgemia.inra.fr/get-nextflow-ngl-bi/wf-illumina-nf" target="_blank">documentation</a>.
 
 show_analysis_paths: False
 show_analysis_time: False
diff --git a/lib/pipeline.groovy b/lib/pipeline.groovy
index 4cc505e..3690335 100644
--- a/lib/pipeline.groovy
+++ b/lib/pipeline.groovy
@@ -140,7 +140,6 @@ def getPipelineInfo() {
     def sn = params.machine_id ?: ''
     def platform = "$seq $sn"
     map['Sequencing Platform'] = platform ?: ''
-    map['Contact E-mail'] = 'get-plage.contact@genotoul.fr'
 
     return map
 }
-- 
GitLab


From 22339c124849a306de4b6770b468ba5ef149815e Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Thu, 16 Jan 2025 14:48:03 +0100
Subject: [PATCH 2/9] COPY_TO_SAVE wait NGSRG treatments completion

	Ref: #145
---
 modules/local/module_NGL-Bi.nf | 2 ++
 sub-workflows/local/ngl.nf     | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/modules/local/module_NGL-Bi.nf b/modules/local/module_NGL-Bi.nf
index 59f2360..1e0aac7 100644
--- a/modules/local/module_NGL-Bi.nf
+++ b/modules/local/module_NGL-Bi.nf
@@ -82,6 +82,7 @@ process NGSRG_ILLUMINA {
 
 	output:
 		path("*.log"), emit: log
+		val 1, emit: ready
 
 	script:
 	def args = task.ext.args ?: ''
@@ -107,6 +108,7 @@ process NGSRG_ELEMBIO {
 
 	output:
 		path("*.log"), emit: log
+		val 1, emit: ready
 
 	script:
 	def args = task.ext.args ?: ''
diff --git a/sub-workflows/local/ngl.nf b/sub-workflows/local/ngl.nf
index 34a1f01..21f7c02 100644
--- a/sub-workflows/local/ngl.nf
+++ b/sub-workflows/local/ngl.nf
@@ -48,10 +48,13 @@ workflow NGL {
 		readsets_created = BEGIN.out.readsets_created
 		ready = BEGIN.out.ready
 		
+		ready_for_analysis = Channel.empty()
 		if(sequencer_name =~ "NovaSeq|MiSeq") {
 			NGSRG_ILLUMINA(readsets_created, demux_stat_json, 'readsets')
+			ready_for_analysis = NGSRG_ILLUMINA.out.ready
 		} else if (sequencer_name =~ "AVITI") {
 			NGSRG_ELEMBIO(readsets_created, params.inputdir, params.lane, 'readsets')
+			ready_for_analysis = NGSRG_ELEMBIO.out.ready
 		}
 
 		bi_run_code = nglBiRunCode.collect().map { it.toString() }
@@ -80,7 +83,8 @@ workflow NGL {
 			project_hash,
 			fq,
 			MD5SUM_FASTQ.out.md5sum,
-			params.run_name
+			params.run_name,
+			ready_for_analysis
 		)
 
 	emit:
-- 
GitLab


From 8db93fb7f457bd52d1748a2917e378453371a284 Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Thu, 16 Jan 2025 15:51:38 +0100
Subject: [PATCH 3/9] Use sample list for readset creation

	Ref: #144
---
 conf/base.config | 1 +
 1 file changed, 1 insertion(+)

diff --git a/conf/base.config b/conf/base.config
index 4bc6650..0a0bffd 100644
--- a/conf/base.config
+++ b/conf/base.config
@@ -262,6 +262,7 @@ process {
 	withName: CREATE_READSETS {
 		ext.args = [
 			params.run_name ? "--poolName '${params.run_name}'" : '',
+			params.select_samples ? "--samples '${params.select_samples}'" : '',
 			params.lane ? "--laneNumberToWorkOn '${params.lane}'" : '',
 		].join(' ')
 	}
-- 
GitLab


From e35d9d0734fbde1ac577ca501ccdb14ab8e8d4e0 Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Tue, 21 Jan 2025 17:50:41 +0100
Subject: [PATCH 4/9] Group fastq and md5 together

	To fit with changes in shared_modules/copy_to_save

	Ref: #147
---
 sub-workflows/local/ngl.nf | 32 ++++++++++++++++++++++++--------
 workflow/short_reads_qc.nf |  2 ++
 2 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/sub-workflows/local/ngl.nf b/sub-workflows/local/ngl.nf
index 21f7c02..29c12bb 100644
--- a/sub-workflows/local/ngl.nf
+++ b/sub-workflows/local/ngl.nf
@@ -14,10 +14,9 @@ include {	BEGIN_NGLBI as BEGIN							} from "${params.shared_modules}/workflows/
 include {	COPY_TO_SAVE									} from "${params.shared_modules}/workflows/copy_to_save.nf"
 include { 	MD5SUM as MD5SUM_FASTQ;
 			md5sum as MD5SUM_INDEX							} from "${params.shared_modules}/md5sum.nf"
-include {	READSET_FILE_FROM_FILE as ADD_RS_INDEX_FILES	} from "${params.shared_modules}/ngl_bi.nf" addParams(ext: 'INDEX')
-include {	READSET_FILE_FROM_FILE as ADD_RS_RAW_FILES		} from "${params.shared_modules}/ngl_bi.nf" addParams(ext: 'RAW')
-include {	UPDATE_NGLBI_STATE_FROM_FILE as UPDATE_STATE_FQC;
-			CREATE_ANALYSIS									} from "${params.shared_modules}/ngl_bi.nf"
+include {	READSET_FILE_FROM_FILE as ADD_RS_INDEX_FILES	} from "${params.shared_modules}/ngl_bi.nf"
+include {	READSET_FILE_FROM_FILE as ADD_RS_RAW_FILES		} from "${params.shared_modules}/ngl_bi.nf"
+include {	UPDATE_NGLBI_STATE_FROM_FILE as UPDATE_STATE_FQC} from "${params.shared_modules}/ngl_bi.nf"
 
 include {	FILE_RENAME as RENAME_FASTQ;
 			FILE_RENAME as RENAME_INDEX;
@@ -41,8 +40,11 @@ workflow NGL {
 		demux_stat_json
 
 	main:
+		fastq_files = Channel.empty()
+		md5_files = Channel.empty()
+
 		// First steps
-		BEGIN(bi_run_code, sq_xp_code, '', sequencer_name)
+		BEGIN(bi_run_code, sq_xp_code,'', sequencer_name)
 
 		nglBiRunCode = BEGIN.out.nglBiRunCode
 		readsets_created = BEGIN.out.readsets_created
@@ -62,26 +64,40 @@ workflow NGL {
 		// Compute results
 		if (is_single_cell) {
 			RENAME_INDEX(index_files.map{it[1]}.collect(), readsets_created, sq_xp_code, 'fastq_index')
-			MD5SUM_INDEX(RENAME_INDEX.out.fastq.collect(), run_name+'_fastq_index')
+			MD5SUM_INDEX(RENAME_INDEX.out.fastq.collect(), 'fastq_index_'+run_name)
 			ADD_RS_INDEX_FILES(readsets_created, MD5SUM_INDEX.out, 'INDEX', ready)
+			
+			fastq_files = fastq_files.mix(RENAME_INDEX.out.fastq)
+			md5_files = md5_files.mix(MD5SUM_INDEX.out
+				.map { it -> [[type: 'fastq', barcode:run_name], it]}
+			)
 		}
 
 		RENAME_FASTQ(fastq.map{it[1]}.collect(), readsets_created, sq_xp_code, 'fastq_read')
-		fq = RENAME_FASTQ.out.fastq
+		fastq_files = fastq_files.mix(RENAME_FASTQ.out.fastq.ifEmpty([]))
+		
+		fq = fastq_files
 			.flatMap()
 			.map { it -> [[type: 'fastq', barcode:run_name], it]}
 			
 		MD5SUM_FASTQ(fq.groupTuple())
+		md5_files = md5_files.mix(MD5SUM_FASTQ.out.md5sum.ifEmpty([])).view()
 		
 		UPDATE_STATE_FQC(readsets_created, 'F-QC', multiqc_html)
 		techno = sequencer_name =~ "MiSeq|NovaSeq" ? 'illumina' : sequencer_name =~ "AVITI" ? 'elembio' : 'other'
+		
+		fq_md5 = fq
+			.map{it -> [[type:'fastq', barcode:'all'], it[1]]}
+			.mix(md5_files.map{it -> [[type:'fastq', barcode:'all'], it[1]]})
+			.groupTuple()
+
 		COPY_TO_SAVE(
 			nglBiRunCode,
 			readsets_created,
 			multiqc_html.map{it -> [[type:'report'], it]},
 			project,
 			project_hash,
-			fq,
+			fq_md5,
 			MD5SUM_FASTQ.out.md5sum,
 			params.run_name,
 			ready_for_analysis
diff --git a/workflow/short_reads_qc.nf b/workflow/short_reads_qc.nf
index 5dd432a..f02ca31 100644
--- a/workflow/short_reads_qc.nf
+++ b/workflow/short_reads_qc.nf
@@ -119,9 +119,11 @@ workflow SHORT_READS_QC {
 		if (params.skip_core_element) {
 			log.info "Elembio's sequencer but skipping CORE_ELEMENT"
 			demux_stats = Channel.empty()
+			fastq = ch_read
 		} else {
 			CORE_ELEMENT(ch_runManifestJSON, ch_indexAssigned, ch_indexUnassigned)
 			demux_stats = CORE_ELEMENT.out.demuxStat
+			fastq = ch_read
 		}
 	}
 
-- 
GitLab


From 5d98d07504c04ea600266e2456b56c9635af51e9 Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Tue, 21 Jan 2025 18:03:19 +0100
Subject: [PATCH 5/9] Deals with new elembio config file and env

	Ref: #148
---
 conf/prod.config | 3 ++-
 conf/test.config | 8 ++------
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/conf/prod.config b/conf/prod.config
index eb1bd1a..0670687 100644
--- a/conf/prod.config
+++ b/conf/prod.config
@@ -3,7 +3,8 @@
 //=========================================
 process {
 	withLabel: ngl {
-		beforeScript = "source ${params.ngl_bi_client}/GeT/bash/loadConfFile.sh ${params.ngl_bi_client}/IG/SystemeInteractionNGL-Bi/conf/prod_illumina_qc.conf"
+		def techno_label = params.sequencer == 'AVITI' ? 'elembio' : 'illumina'
+		beforeScript = "source ${params.ngl_bi_client}/GeT/bash/loadConfFile.sh ${params.ngl_bi_client}/IG/SystemeInteractionNGL-Bi/conf/prod_${techno_label}_qc.conf"
 		publishDir = [
             path: "${params.outdir}/ngl",
             mode: 'copy',
diff --git a/conf/test.config b/conf/test.config
index c2de5cd..a795043 100644
--- a/conf/test.config
+++ b/conf/test.config
@@ -10,7 +10,8 @@ params {
 
 process {
 	withLabel: ngl {
-		beforeScript = "source ${params.ngl_bi_client}/GeT/bash/loadConfFile.sh ${params.ngl_bi_client}/IG/SystemeInteractionNGL-Bi/conf/dev_illumina_qc.conf"
+		def techno_label = params.sequencer == 'AVITI' ? 'elembio' : 'illumina'
+		beforeScript = "source ${params.ngl_bi_client}/GeT/bash/loadConfFile.sh ${params.ngl_bi_client}/IG/SystemeInteractionNGL-Bi/conf/dev_${techno_label}_qc.conf"
 		publishDir = [
             path: "${params.outdir}/ngl",
             mode: 'copy',
@@ -18,9 +19,4 @@ process {
         ]
 	}
 
-	withName: CREATE_READSETS {
-		ext.args = [
-			params.lane ? "--laneNumberToWorkOn '${params.lane}'" : '',
-		].join(' ')
-	}
 }
-- 
GitLab


From 04a8475af3fc8625bc043af0c3931512e32b7545 Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Tue, 21 Jan 2025 18:15:42 +0100
Subject: [PATCH 6/9] Ignore some error in analysis creation

	and more time to store fastq files into /save/ngl

	Ref: #149
---
 conf/base.config | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/conf/base.config b/conf/base.config
index 0a0bffd..0be9913 100644
--- a/conf/base.config
+++ b/conf/base.config
@@ -255,8 +255,13 @@ process {
 //			   SHARED MODULES
 //=========================================
 process {
+	withName: STORE_FASTQ {
+		 time    = { checkMax( 3.h * task.attempt, 'time' ) }
+	}
+
 	withName: CREATE_ANALYSIS {
 		ext.args = params.project_hash ? "--projectDir '${params.project}-${params.project_hash}'" : ''
+		errorStrategy = { task.exitStatus in [6] ? 'ignore' : 'retry' }
 	}
 
 	withName: CREATE_READSETS {
-- 
GitLab


From e56bc9b3391bcfada2be4fb77494cebd0c0f44ee Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Thu, 23 Jan 2025 16:58:51 +0100
Subject: [PATCH 7/9] Remove unwantes graphs in multiQC config

	Ref: #150
---
 assets/multiqc_config.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/assets/multiqc_config.yaml b/assets/multiqc_config.yaml
index c981be6..c83e60d 100644
--- a/assets/multiqc_config.yaml
+++ b/assets/multiqc_config.yaml
@@ -49,6 +49,12 @@ report_section_order:
     summary:
         order: -1001
 
+remove_sections:
+  - fastp-seq-content-n
+  - fastp-seq-quality
+  - fastp-seq-content-gc
+  - flash-histogram
+
 module_order:
   - fastqc:
         name: "ReadsStats"
-- 
GitLab


From 4effb395fbe33a02a6adf2c659c03cebba11573f Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Thu, 23 Jan 2025 17:04:41 +0100
Subject: [PATCH 8/9] Remove from ngl subworkflow md5sum and readset Files
 creations

	Ref: #151
---
 sub-workflows/local/ngl.nf | 33 +++++++--------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)

diff --git a/sub-workflows/local/ngl.nf b/sub-workflows/local/ngl.nf
index 29c12bb..f05c720 100644
--- a/sub-workflows/local/ngl.nf
+++ b/sub-workflows/local/ngl.nf
@@ -12,16 +12,12 @@
 // -------------------------------------------------
 include {	BEGIN_NGLBI as BEGIN							} from "${params.shared_modules}/workflows/begin_nglbi.nf"
 include {	COPY_TO_SAVE									} from "${params.shared_modules}/workflows/copy_to_save.nf"
-include { 	MD5SUM as MD5SUM_FASTQ;
-			md5sum as MD5SUM_INDEX							} from "${params.shared_modules}/md5sum.nf"
-include {	READSET_FILE_FROM_FILE as ADD_RS_INDEX_FILES	} from "${params.shared_modules}/ngl_bi.nf"
-include {	READSET_FILE_FROM_FILE as ADD_RS_RAW_FILES		} from "${params.shared_modules}/ngl_bi.nf"
 include {	UPDATE_NGLBI_STATE_FROM_FILE as UPDATE_STATE_FQC} from "${params.shared_modules}/ngl_bi.nf"
 
-include {	FILE_RENAME as RENAME_FASTQ;
-			FILE_RENAME as RENAME_INDEX;
-			NGSRG_ILLUMINA;
-			NGSRG_ELEMBIO;					 				} from "$baseDir/modules/local/module_NGL-Bi.nf"
+include {	FILE_RENAME as RENAME_FASTQ						} from "$baseDir/modules/local/module_NGL-Bi.nf"
+include {	FILE_RENAME as RENAME_INDEX						} from "$baseDir/modules/local/module_NGL-Bi.nf"
+include {	NGSRG_ILLUMINA									} from "$baseDir/modules/local/module_NGL-Bi.nf"
+include {	NGSRG_ELEMBIO;					 				} from "$baseDir/modules/local/module_NGL-Bi.nf"
 // -------------------------------------------------
 // 					WORKFLOW
 // -------------------------------------------------
@@ -64,13 +60,7 @@ workflow NGL {
 		// Compute results
 		if (is_single_cell) {
 			RENAME_INDEX(index_files.map{it[1]}.collect(), readsets_created, sq_xp_code, 'fastq_index')
-			MD5SUM_INDEX(RENAME_INDEX.out.fastq.collect(), 'fastq_index_'+run_name)
-			ADD_RS_INDEX_FILES(readsets_created, MD5SUM_INDEX.out, 'INDEX', ready)
-			
 			fastq_files = fastq_files.mix(RENAME_INDEX.out.fastq)
-			md5_files = md5_files.mix(MD5SUM_INDEX.out
-				.map { it -> [[type: 'fastq', barcode:run_name], it]}
-			)
 		}
 
 		RENAME_FASTQ(fastq.map{it[1]}.collect(), readsets_created, sq_xp_code, 'fastq_read')
@@ -78,18 +68,10 @@ workflow NGL {
 		
 		fq = fastq_files
 			.flatMap()
-			.map { it -> [[type: 'fastq', barcode:run_name], it]}
-			
-		MD5SUM_FASTQ(fq.groupTuple())
-		md5_files = md5_files.mix(MD5SUM_FASTQ.out.md5sum.ifEmpty([])).view()
+			.map { it -> [[type: 'fastq', barcode:'all'], it]}
+			.groupTuple()
 		
 		UPDATE_STATE_FQC(readsets_created, 'F-QC', multiqc_html)
-		techno = sequencer_name =~ "MiSeq|NovaSeq" ? 'illumina' : sequencer_name =~ "AVITI" ? 'elembio' : 'other'
-		
-		fq_md5 = fq
-			.map{it -> [[type:'fastq', barcode:'all'], it[1]]}
-			.mix(md5_files.map{it -> [[type:'fastq', barcode:'all'], it[1]]})
-			.groupTuple()
 
 		COPY_TO_SAVE(
 			nglBiRunCode,
@@ -97,8 +79,7 @@ workflow NGL {
 			multiqc_html.map{it -> [[type:'report'], it]},
 			project,
 			project_hash,
-			fq_md5,
-			MD5SUM_FASTQ.out.md5sum,
+			fq,
 			params.run_name,
 			ready_for_analysis
 		)
-- 
GitLab


From e9fed7ababa4f22b1d5fe5683b7d430efdf13e09 Mon Sep 17 00:00:00 2001
From: jsabban <jules.sabban@inrae.fr>
Date: Mon, 27 Jan 2025 09:33:42 +0100
Subject: [PATCH 9/9] Update manifest version

---
 conf/report.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/conf/report.config b/conf/report.config
index 42aef54..7cd6657 100644
--- a/conf/report.config
+++ b/conf/report.config
@@ -29,5 +29,5 @@ manifest {
 	description = "Workflow for Illumina data quality control"
 	mainScript = 'main.nf'
 	nextflowVersion = '>=0.32.0'
-	version = '1.26.5'
+	version = '1.27.0'
 }
\ No newline at end of file
-- 
GitLab