From 13df59bfaf45d385fc7d745b31b9a174731da0ef Mon Sep 17 00:00:00 2001 From: phue Date: Tue, 12 Mar 2019 17:57:31 +0000 Subject: [PATCH 01/79] remove singularity support reasons: * the kernel running on the compute nodes does not support overlayfs und thus containers downloaded from public registries won't run because they lack the /lustre mountpoint * building containers manually using the Singularity file would work, however nf-core pipelines will stop providing these files in future releases --- conf/mendel.config | 2 +- docs/mendel.md | 18 +++--------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/conf/mendel.config b/conf/mendel.config index 940a914..0715b42 100644 --- a/conf/mendel.config +++ b/conf/mendel.config @@ -11,7 +11,7 @@ manifest { } process { - beforeScript = {'module load Singularity; module load Miniconda3'} + beforeScript = 'module load Miniconda3/4.6.7' executor = 'pbspro' clusterOptions = { "-P $params.project" } } diff --git a/docs/mendel.md b/docs/mendel.md index 3f3db20..ae0ad4c 100644 --- a/docs/mendel.md +++ b/docs/mendel.md @@ -2,27 +2,15 @@ All nf-core pipelines have been successfully configured for use on the MENDEL CLUSTER at the Gregor Mendel Institute (GMI). -To use, run the pipeline with `-profile conda,mendel`. This will download and launch the [`mendel.config`](../conf/mendel.config) which has been pre-configured with a setup suitable for the MENDEL cluster. A Conda environment will be created automatically and software dependencies will be downloaded from ['bioconda'](https://bioconda.github.io/). +To use, run the pipeline with `-profile conda,mendel`. This will download and launch the [`mendel.config`](../conf/mendel.config) which has been pre-configured with a setup suitable for the MENDEL cluster. A Conda environment will be created automatically and software dependencies will be resolved via [bioconda](https://bioconda.github.io/). -Theoretically, using `-profile singularity,mendel` would download a docker image containing all of the required software, and convert it to a Singularity image before execution of the pipeline. However, there is a regression in the Singularity deployment on MENDEL which renders containers downloaded from public repositories unusable because they lack the /lustre mountpoint. - -If you want to run the pipeline containerized anyway you will have to build the image yourself (on a machine where you have root access) using the provided `Singularity` file in the pipeline repository: - -```bash -cd /path/to/pipeline-repository -echo 'mkdir /lustre > Singularity' -singularity build nf-core-methylseq-custom.simg Singularity -``` - -After you copied the container image to the cluster filesystem, make sure to pass the path to the image to the pipeline with `-with-singularity /path/to/nf-core-methylseq-custom.simg` - -Before running the pipeline you will need to load Nextflow and Conda using the environment module system on MENDEL. You can do this by issuing the commands below: +Before running the pipeline you will need to load Conda using the environment module system on MENDEL. You can do this by issuing the commands below: ```bash ## Load Nextflow and Conda environment modules module purge module load Nextflow -module load Miniconda3 # not needed if using Singularity +module load Miniconda/4.6.7 ``` >NB: You will need an account to use the HPC cluster in order to run the pipeline. If in doubt contact the HPC team. From aaa7be049a9589e79cce863fd6d582fb8fcf7799 Mon Sep 17 00:00:00 2001 From: KochTobi Date: Sat, 16 Mar 2019 14:11:59 +0100 Subject: [PATCH 02/79] Update cfc config * Change singularity version documentation --- docs/cfc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cfc.md b/docs/cfc.md index e1cccac..5cf87d1 100644 --- a/docs/cfc.md +++ b/docs/cfc.md @@ -10,7 +10,7 @@ Before running the pipeline you will need to load Nextflow and Singularity using ## Load Nextflow and Singularity environment modules module purge module load devel/java_jdk/1.8.0u121 -module load qbic/singularity_slurm/3.0.1 +module load qbic/singularity_slurm/3.0.3 ``` From 441ddbcdab01871842451bf4b33d757d8c63d87a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 28 May 2019 15:25:10 +0200 Subject: [PATCH 03/79] Newest Update is 3.0.3 --- conf/binac.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index 5df94bf..0e69d4a 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -10,7 +10,7 @@ singularity { } process { - beforeScript = 'module load devel/singularity/3.0.1' + beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' } From 66eeeebe81d1aef1d8083e492484e47975d8760b Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Thu, 30 May 2019 11:15:38 -0700 Subject: [PATCH 04/79] Initial commit of CZBiohub --- conf/czbiohub.config | 98 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 conf/czbiohub.config diff --git a/conf/czbiohub.config b/conf/czbiohub.config new file mode 100644 index 0000000..ff1523e --- /dev/null +++ b/conf/czbiohub.config @@ -0,0 +1,98 @@ +/* + * ------------------------------------------------- + * Nextflow config file for UPPMAX (milou / irma) + * ------------------------------------------------- + * Defines reference genomes, using iGenome paths + * Imported under the default 'standard' Nextflow + * profile in nextflow.config + */ + +docker { + enabled = true +} + +process { + executor = 'awsbatch' +} + +params { + saveReference = true + + // Largest SPOT instances available on AWS: https://ec2instances.info/ + max_memory = 1952.GB + max_cpus = 96 + max_time = 240.h + + // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + igenomes_base = "s3://czbiohub-reference/igenomes" + + // GENCODE (human + mouse) reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + gencode_base = "s3://czbiohub-reference/gencode" + transgenes_base = "s3://czbiohub-reference/transgenes" + + // AWS configurations + awsregion = "us-west-2" + awsqueue = "nextflow" + workDir = "s3://czb-nextflow/rnaseq/" + + // GENCODE GTF and fasta files + genomes { + 'GRCh38' { + fasta = "${params.gencode_base}/human/v29/GRCh38.p12.genome.fa.gz" + gtf = "${params.gencode_base}/human/v29/gencode.vM19.annotation.gtf.gz" + } + 'GRCm38' { + fasta = "${params.gencode_base}/mouse/vM19/GRCm38.p6.genome.fa.gz" + gtf = "${params.gencode_base}/mouse/vM19/gencode.vM19.annotation.gtf.gz" + } + } + + transgenes { + 'ChR2' { + fasta = "${params.transgenes_base}/ChR2/ChR2.fa.gz" + gtf = "${params.transgenes_base}/ChR2/ChR2.gtf.gz" + } + 'Cre' { + fasta = "${params.transgenes_base}/Cre/Cre.fa.gz" + gtf = "${params.transgenes_base}/Cre/Cre.gtf.gz" + } + 'ERCC' { + fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa.gz" + gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf.gz" + } + 'GCaMP6m' { + fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa.gz" + gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf.gz" + } + 'GFP' { + fasta = "${params.transgenes_base}/Gfp/Gfp.fa.gz" + gtf = "${params.transgenes_base}/Gfp/Gfp.gtf.gz" + } + 'NpHR' { + fasta = "${params.transgenes_base}/NpHR/NpHR.fa.gz" + gtf = "${params.transgenes_base}/NpHR/NpHR.gtf.gz" + } + 'RCaMP' { + fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa.gz" + gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf.gz" + } + 'RGECO' { + fasta = "${params.transgenes_base}/RGECO/RGECO.fa.gz" + gtf = "${params.transgenes_base}/RGECO/RGECO.gtf.gz" + } + 'Tdtom' { + fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa.gz" + gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf.gz" + } + 'Car-T' { + fasta = "${params.transgenes_base}/car-t/car-t.fa.gz" + gtf = "${params.transgenes_base}/car-t/car-t.gtf.gz" + } + 'zsGreen' { + fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa.gz" + gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf.gz" + } + } +} From ac804859f42a2bfa20a7f9c6fbbc27915729ed47 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 31 May 2019 00:47:06 +0100 Subject: [PATCH 05/79] Add nf-core logo --- README.md | 4 +--- docs/images/nfcore-configs_logo.png | Bin 0 -> 14072 bytes 2 files changed, 1 insertion(+), 3 deletions(-) create mode 100755 docs/images/nfcore-configs_logo.png diff --git a/README.md b/README.md index 6a53642..75d6199 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ - - -# [nf-core/configs](https://github.com/nf-core/configs) +# ![nf-core/configs](docs/images/nfcore-configs_logo.png) [![Build Status](https://travis-ci.org/nf-core/configs.svg?branch=master)](https://travis-ci.org/nf-core/configs) diff --git a/docs/images/nfcore-configs_logo.png b/docs/images/nfcore-configs_logo.png new file mode 100755 index 0000000000000000000000000000000000000000..eef073868983145737377f81cb92f40132f3c866 GIT binary patch literal 14072 zcmaKTWmp_d(>3ny?gY2s9(?ft2`(Gl-QC@mU_leIgy8PBIKeHrySwYVxu5U%x7Vys zb$9htcTJyjI_9IA0wx+68Vn2!rXoOA69xu$3i^2v1qu3Us1FQ+z9G0tD}F+OKKxP4 zqoLnXodJ4oFfd3}|DLdJf}0!AA4%QibltU_EZx1nxLUw?d3kZ#ING_HeQ~znbaJ)I zJ{2Q_fuVs>l$HAAopX}wv^Ebr+^Rq}@wV^zLb^ES_C?aY3|Jw06oT z--@bqr2Vd}UUVFlVkb-H4%8+Vk6r;$v?0k`6_HV$CFoG zgFsFrWq6~;-vI1U2nC~pg2?_QPKq#k?+Rzt`OFtxHzv#`B{)Rd^N*%Wc{lU4xY^sMclgQ7~(ocU~sUpEhzIz_|mh^Ri zwT||GjZd-MuKGgB{>lzyg4yCdT=>RI%{hDTD_;Wb%T$4WW9iu#IIY+NS*f3%P+yfm z2Ep)f%U<-(P9Y94X&`!qUnez;Yh!_S0b3?o_T)1lR;U?KD-1DwF~yw&EL@RuD*JFc zW#3-uB(aJ2T^MPffK=muCB-=0U39A55k_zpl7<+%S zjXT+V2-LAt(ZSXvNw;8Ep87UZMBUN46%|i?Mu7N>Id%%+GXxR#ti3Qfk^uoe=i^%f z|0}Nx=??_pJh;+pUbDd>m50LV3xj9&^Uqyf_*=rQZ+2Gs?=K!g$%~nl?DjO*lVDH% zVDC+#&gAw)VlPbBt}ZP&=}6qIj^qjbstT9V0SnK2JT?Uy8|}t&*b8G@J2@DSiiQbC z#`_R_R48Fq(&fbTUZX~{m6JX{$Fey;!7z0$_?}p&C%-IEVee2mLMZUHnarBFQ9QPw z&LZW=^jn1g>f{4%^KMPo8?VUddKAPKFvL3HvhVCvLSmr}VDJriEll9>)~2+9m(Z(S zM+RagE@44SLvp9_9vU8v;A2pXrx1RR>F#*`Gq%ZILm|uJ{MtplYB#p&31HiH3Aj+>X`kKQEMxd;%AqIi2DNf>8U^dr z2ywG7ROi`nOPVC?mgh4(e)?we`XgN=LF#3svcc;mHRH2f$LB)@5rotgCqS<+5o<-q zC=UsG&7Sw1agEo?dUWSxe5vp0`#bA5Ff1Y4 z)vaL7TRiSHfsSVBe`DT96Vw00scw1#WDGJCYCu36_RFv-n?D}jkIXcPxovTz=wvVO zR;{(K)N^2JygQPg9*Lvm zqT`&wiqN6qAOeT}hqBx*IYFAsnAV#X?(ThFO*_dV+TvUjK*|tn4TE zB}xAz%da=j_`7+od!B@rd@ph&>+`31fO2S9;&RgH4h?Ii2IH?>#Z=Z;=84>XF*_b- z(d;NPNk43Med{tQIr;dPCyUCV@M5$FzR0*~rJ!_FbgJX4S|vh@Fn`02ldN4r0W%uX zy)VQMpW1KIYw944TrgcSbYoknf85==(I9-q&=dN-?U zJ*r|JXhw0S;AVgXxCIZO*(+Nwrg?S1laMx2;?|%Wum&xiBRi~uel>pFnVG1HxY#l) z4k%()Dt(vF$`uocgqh{Lp{MJow(KdJ)c)OfaX#R%lP!~`Cwi$rt^8o)?pCaozTb05 zce+b=2!_3^r-sww*mwNiZjp=Q?K&aODXH){$t1m4<<_A3uNLezyeHgZ&uq-7Jdm=A zN;h;yR|Ty-WFvZ1lvcQh5)@-2+Iy?dK>39SkMF&xpM`1FTv{H$BcWpOLP2Ny#^J*& zXLbnU;E}!-L-tUxc!*JL!;S11-Y{;FCVZLvDwkWxU5!9GnhXwZq+uEStD&w|H;?&Z zX5u#t!Mdqr?Q!#wq>xFR5c(6zn`h3Q{w1!kg(V2Ej5NMlW_DhMdQdIHUfYE3iCX`y zKB2NV7weBmEXYS8 zdJA?)=XB>Hm+S0&FF4srh*$`ZLbtWB-sGq_J{aB!+hxop!95{dFi)11{;p3%@Z)en z3Iv|Ba@I?Z5FH$3UO1QChbkOzBySjRyx(yX4ZoXnfC~gc@(4gYEJWEF>zKRTC@+1bZ7YtlcX+>%>^m}=B^iCx;R|H zkCSd9)rBie+kqT~2^&#vYPsG4Y6>FWk4;p8M!Q1XOrbA$6F*UQ`N2p{?FH#?erNG9 zYD1KV05da%^`dBa4z6=uQuBHj%5(q04taP<1uE@6AmzE;Hf-k)h}j8D4rUJWvBr9} z@47bmmG$;B7XXsL78k6XOz5t8Pjuc2b0?0L^ATtu&EtHS=|rMJ?!CK^u^ZoVmHe*J z(<_&+qBdhJCz5CH(c_y;2tmY}g+rq2`Xz@AukCsrkBJ>CskR~oi|o_Yz+kggb-x&v zaHX3ESUf}$B!0(CM;T%l>{dIcj7YNeVO?n(mID02ouW?RlkXTux2`D6bn@q`9fG~+ z?~H9_lv04Uu=mcE%K`+*Zjd~6|MC}srOz_2fg8bxQ3K1zX%BJSDhPf;iVqW{tzR!j zx8RH79q5C;rxa4VQh8AJxfN(|Q+S7MicUsi$i9P>`Y90R%ze&x{@(e$;)9secm;%w zU?4_dn+_E8`y$&uNHm#Gnuy$EIHx%!IS&3-+n*?!oic!PVUN?6ZaQl@^4=9 z-=f56#O&)`KQqdTXy*u+0td#+Ub9@(5~o~#w8izcS(jqq;)_{^X;bcQyp;Y3tMV?8 zNq<}pfN(!UmqgCDv!lj<$v#Q4MuoGJVqbZiRx9su(&U{1osvBqjCQO49*K(ATfU&g zsrGh{4=mO5Q5JJM&V#EvofEb{)BR+_d&sOKG{Wf!+pc*>h@BZNJ6DmokS=}ByZ*f? z!Me8`AaPtmmoOIr-0LRYG~hwDdiKVyJa69{yws(Ssf%v^9CrsqC zOW~T^?7T4gNgGFfUu>BZQP_#`P-2~o>SA7qtRCxy3ppO}lkN3~g4nNWCP1ycuRJru=dKA zen)kjds_dBA{{N8`koLsZNI(VI|yd4151)#&KV-jroHVIn?*dtR-^tnUIeP`oFO02`(2kI#u3l`ZD3yc_P)QfO}&x0bFqE+G;bqr zbj_xS)UM)8V!tGV{P|obtSE71E#vorLu|N?@2?es)q3B zQj9e`25}~9mngU3PLcR4fvQHC*2q+P(C@S9y7r{?ijgbT?JqkQ>=M4z^gZ$NdpN;;w7l)g8g3|6wLLCV>8B8V ztBau)oV%NP=seuAIDA#bm#|ed+Wo6#8CVyDKMJ#NqR{}{gmfG&6}7DxaKG_l`+aFV zEY0Rp6%1?>v_4ZGtDQAYQaLTYR#D|+)3oal^|}tXk#frS>OZjymc;PC4=qNzVwmO2 zvq0!KZ-X5~)@I?%h_sia24VU0jC+!U79Kj2c^lwEu0m>2IzIc!=a~s?b3qWdi)iga z{E}_CwVb~yA17FEN^DLv%+JtZHM2b8l(i^oEh{-Qp7#L2bN=zr6du#~%6er30|P(1K)+Le8Pyn)lh zweg2eUD=#?V4EcQDVf;IGLwVR!WugT+Aq#oirVI))q3t7p8@<|2wJQ>Kdu~NXeXv- z0cyYKyWTyBO!nVL=d22uTlkVTN3>y{!Xct3{|za{d6En^5otgyme4xPiCJ3McQua_ zGdXm|W@Q8Cod}|&U%n67jyV#HyEq}+a@)+&z*iV0tIx$>89LVRQoCJehWBz(DN4-Z zZ?Ry~S8oiFd={L4cLU;D`{CzNxaaVuNXk^NfG96(`eT zFt+;RaFM9T!UIZG=fd{WxB0LH2_c{EXK3Pk#X5d#uDes_@~YPJ4A_5`GG@KWU-r_U z?GUT9o$e;gue2AFDXF2TG|Kwa>e@C0J>EkP&6Aw02a9hNYV|7@CFp2dwq4d; z3{4}A2xNFHS&B_wFgnUMxS6qR=JoLpIEd4&;-rrbvi3yK0?t1dPhkl@0OC^rpob}^ zWwZyY(GEDoRwKkQ=gHd;GIF6AA!GUCu?RL&DtH|dY_iCB`m_eg`Ha<(pnvt% zHd!#Fv8CC~aotgfHfY^)I6a}tS6&Q8<#zlRX9FhNAx&Y*-#QwR zigLF_REY-L#d3!Z{i@0uejKU&w@D0iEkU2BcyX4&*>!IO;5ACshzu2eb zTJ3CmTLH$V%Y14XsIjG6c8OkJT63CcQVU`~iC26Lb}hht)m49`qX?rXX{41ZZQHl4 zLu_%X2>2o=_aGwOH|Q;>V4-eGQG8=>R1*i^C(ElwVCE`1@yM}ajhgadl|YeY@Kitd zUCK8*ctfoQrScXvnMctE^6d6a$j!?tK~D$>_S8WBp5jD1fmEQw@*&601Ub?9_>1RV zh@`)~?Mf7rltJr|C*PiINcxzpYStb36+~lbq^xdkn@B65f(|-qjRli%fy1Qt&y{_uRi5%aLsnnpZCW1-<*B zq%OL=dX?=wPU(>Ny#$aDqeLx)rR2g-nRa!iR+XBeFY7J+C#1STcruA5fjV)5zyllu zMr5*`TGd9j&2sry)>lql?%Ja;v5(yzAvSySjxwhPv9m=-b^y2^C852tjFZp#CbCu0 zJ5km~eW+-WXGIdOw$^$J0g%6yayH7QV*tW)P5$fvO(*-D`risCDfQyRB7##lcQ{rH z6=jp1H>J~MC^A0hlBF3z5$JPKN^ud0vEj;N?1Qm6*>7ZPYaGVV)13ej&AK~C73FZ=M^)Pn}E*w zgT1P{H+5PlL=G<^5!1I09+``mec-D%ArgCL*vMZK-+Wt(qEJ8$=6PooQRA6XR@SyzjgG}Tr|4#h$ zC*yPCNe50b;VFA}`Q;E!0jQ+ZD$}f$;kQh44P0Z}hz?4+ND7}Xj1_WICDP;%?Ms@I zryWiy?1Ok}pGl0B#m|=(>1M560GGoi7s635PKh;)ip__@9I}naRhLhdF}ZOlM8#sL z3(yn0%r80KP(Dz88yr9U3xayU`g#IOENc@@Uy;?UQSrNVz2onTakwV|xz3UogCXsM z5CGl;+q?4cP$ch%P#AG5OqI zgU{qGjDIAbqwv)kv;Y^$$B7QxZceDb& zMcdzJnY9x3lDp%l2W=V8yvVLA<1T7i*9|F$RL_or9=+QO8ssR8Q@DsS$qHUD2`Yr@ zVN2YWuh34LGbKIk=aPSh&{Hl+MEFUaGgOgzcU6}oFDD?JGvo$@$iOe(v2Zs-8tRP8 z3>?1L6(s$ILcdKo{h$WqJcdPHg)sZ}4y0$gf%a^dpv-(2|AlG}z~ZQrOkEJ2>=O7K z5Zw>}a`hLLH&>Qn4X(sf<=hj5U-e=;WxEzg;yK=$8tW;c|BTla7rtbAoM3-tBqQ`t z?+CGg7(n=dvJlTN@8Gb5r;(fJL@Vs-PT$c?#S-$kP;baGM$)&U70H`nJ;8LsO2EF# z-=Az5L%zYDF2$!RYf)6ql2LdylqGbafkZU>A~4Q}lLJ!WC8fp;oRHM$bZcI5fEb#L zTebE}?Uxi!u*=lDM#&trZm3lLd>Hq{8``zlxaBmmhU~DXs0#yQgWzowh#o>|iKaSf z4f!Q6SobL#;uY;U>`l>i{rimREfiNNouB%-O>&)}J`TJo7mN0uxnddfIT~oX%rQx* z_t`l8C6Kd5L^nbxQKYx)76yry_qD1_y5W4x!+wp@=OqI!2=Ny@I@yXAA_ZbR+!$@n zNKT>0LDee}iSm8SGwy<&cn#YH5jV2Nntq(hYzv(zu_&!ohM!gwE1v~~5%%ma%}<<} z!f8LzDD?S3w{B>Bj?KX6z}?9&pM6+Pjk?^?@9m0US@s3qwvg&=aLOFU-%4p$Xz&n5 zx0%Ytk4Q6Igr*Y>%OT5-3q=2NpcPI4Mx5mji2EACLZ~-#5-drq*T_vABNLKTQZyPp z)3}!vt`qBY{e#O`e-=>I>cax<;wu@(luXetxOm-Nz{()=vE_ zFQTWTP!hg#1ZitZaNd3z{AQH$fO~4IqG*X(gQ-7sAcnbsEyN57Ku?AU^?^QL4S01S zV6oeijVHPLK{+10^m>R(>M!kmrFH3L^g!Y)g%I*Bfr4Nag?DJ>A>T_UB|~NY;90N* z8a-%)i>Aap=X`=1AyU)f7K1XFSmBbAN6M!gL42`bk%!++L``gt^wLx0LiCUT@(@ud zAaFD=&%M;LAOU4@{%ljOJ1e=7U;1u`Hgo zZY%?)sw`+4k|df0zJAF&fleT}JhM4CdR7OYjF(zl)kxib=d<`xl7w7f+uw_NfR$($ z5DbxzL(4R&kqb)H@}=;WI+IX@a!}jIG}f{~ZqtE-VKq2?bs6RZB+;^1T&xh1LM0A5 zsb=&V2mRlsy9PKb`)t9lgO0u7VvE$t?XEmY&ZaMgf1t}~_;*Pwj3uYDf#*k5MC4r$ zzkyXD9+jokR8Jn`nL(xpUme;L*bfAT~!%VfwdRm;8kZ|DsD6AIGG~nEf>#vuXGs9vQR3{jTdvd087*nFkl&P|EGP&L=t`8bS@>W(>Hm!85G8BW=q?n`nYS|u>b#iVjZ^PVI_ z6SkP_5x-W3)tf5b3a-^6v0~6~*=r9es;rv^E_>&CqKgoQ3Ce}gvxGJ)5jy=w4?x|O zZ?7V!94ldfeBIldC=4}impn%`l}4K_CL;2s>3V8)Pr?QzFs{!~{O+Vf@O@z0^tQ{+ z^3mhN{h67~+WDS@=k9drT;~?Kft&rF8WVi(QWZf9ENiX}r*^9MSdK z;#-OM^V|DfKR5Zc2Um&qd8_^+9o;vsK}RB@R(}nL-9iK6^7Uf=qt5VSTNkfZ{U*dZ+zztRB9XVl&33j?|LB}_&WU*DgJuzmie@9k5Z+Zgz59Q_{!&8$E- zcERFN6oyQh6;+AC~Q$ZaJbxm)b1Gz=U?A zlG^1@(*27asw+pN+9r#wOZa=&`z;y7hVC1ApBsAQOag=CGg8ljs7Dy9PKFv;;`!II z#C^M|Kj?s=9LR&~yRNYtrskF~7#Jd(e_DY2?+GFrU7q&7<(7$E^JeokE*>|jJnH1$ zf5{|nG1!jEs_$3z&7M#De>$b{JW<`fJ_l(3eUL%@IJng4-fVF(RJqz>zH`ab-n!!D zc{kQxB6jvMiz`>BwScpB>5OR9k9%i{zUvde+uCjA*uBA4%zu3J_P`G8ncXxCmplWm zK>>Qde_yu_jDNN&{YA~dZz2KxVR^~7pAKtNX5E`UOq^?M;?4XN>}mR1JUb?uPwfVA{${Z9M*Y+9ix|FCF}3NzXPV5ZAS2zLiwT zYv6zDYdiYbo~MsrJr=X)L>M$F1|{}W*AWPTO);}3&fn$Y_n0`QBsBex#w*|S|Iv9$ z+@1*X$W2a^lSBzzs_z*MiiA)~A3aVbYhTHs_h|%jsPYP-xk-U4mzQ~QV^QBcsC9}& z9Qp;w_qAF`@&X&AhT=v3)dBSXV3Up-AH_>G!tHU)7M-GZ)Gc@?@T~nG^`06ET(53& zh^83!bF;k8T|HO$pA;Yz_`bMvx&_Qr?sTthRxID>ePgi;;{t;@xn!_-iUT*u}g{Ex@K<^jdq>2 zWb%gsvj52lQbRTWcHEMa3#3!q0|Cno+keo)9{O{%m{uhlBeLD){wpGtbYQu*OaIIq zC>6fqSbZJ}g1gnYMgM5V(dct9GXaGJ_%ymK4eH{v{%IcwJaZ)~@{5PXNCPcwO)jSo zYIk$ppR|!h_BYV6s%D=g3M6(8mu5qo@$=V4gdMt>q{xfup*^Y*+n;P>CNlf5YbF1w zKoiSI%;589x6OF7(}jr{u2LqGDy}?1Dj^V1!jb7~ zl&~;gIyzR@7KRolL z{|b!(dacva$T;sU-1`>pYqP~pl}z}8l~_`c`r##EhGMx_>meDGiw}@j+$CsacSBp8 zN@f&X3>l3{p&tQ(@!OHHk z@5orTf3)H1V)|#bTf#cQp220uIW*BD`2@7Wh{6RVf5>wEu63KII=E&{dFh)0=LCNR z0?e3Bb4Kzg=Q=_h0kP^%+fK_^KI&o5lsfc5dkGNJ?H92)Z^@+Q{cwMIQJS-b(Lpy0 z%Bm3arHVS!Zjq@k5EckebhtqWD1qpnBokH)`I+4NYxghAYlMzihKHoyfIh8p%ii08 zVC3W|O}zc^XVqI7TN^Ayq^(!qIoLCpO4<;yaom-XGeA}njaNeY3ckEMBo`X3zb^oSIZZ51*h;>qKV$}=s&OHxkh72-+NToOA!&o{{-9uZw`DgR=MI@QYWO*kI`nQw2;v}kg1<(X*dUk* z6bj2lY#rZ^fy9YC_Cb64iyn-Mv}?#({C7z?R5`LH0^Lx*Q2&lOI_QvpJ(zUe_wg8h!TNrvDBmT>0FfSBLHRTHVSm;q~BXldyuiK}GIVqyJ0#Iqp;|ed)6F*nE#M^?C#4oId3NIo2DostZC*8r2;X$bLl5N5+l(0eA1F zQn~Spa;%e@L$a>Z`}V`b1>C6?ypm8UiGMnBnT606Ni(Os9kg1USm(njqaEto0D28%sl-6^8U_+DS1aV0b-AdM^oZtWl0CWO6^;&4JCYt!L`CN&SN91h6_ST;9m z3|ft*WhX-<`u79H0(U!P!8I3ru#8k8Q&I)8zo@_P6^L>nvqjjyOkX zL(TBQcJTLG3%u3t!Y{O@dYA@gC3`+kaC=BA;1#)vMNO)K@aq^BB+Z3f17R$2DHv}z zBbrQuCHV|%SP!6-RZvGp7FBH`<9L${zT852tcpHfn}%2o&ZZ^VYu!o_cz1lbM_I0f z@1zxG1w;V3=|~@>mLq$khr`{4&g)Sebm2yj(rM53(7vOm$n%GvpX~%(wgnjuUKqoj z%FO(hkT!=M2E{0-Gk865o0}T3e#I^9wb=(iwUhEQXJiaIM;6;T)};A%exBpTqAv55}-tc%5vN#7o_`^MFou;lq-f~&f1$nL)1njB8p2;;&j2#-w*>Ivk% zyOI4gRM8y(mzzZK#mk)Tu-g)dF`DmCAq$2zfjO0WSNbz~CEq`{y+Ftfc?0|0iq@y- zK_Xm~mU^FMQ0DuWe5tX;K;rJ*8$h0M2SvOJmf=9ILLeK_9Pno|L}!qTv5+@Knn1wn zreCbz9&7Gx3yQ?H=3P-Zc#Dp^h~OJ%0$DXPX{aCR&hY1W!qnUov3cY(0q178lxLAI zHdP=zVI!QV&Bu3lxRW8wZ~-DTrS*cf0mv?3YSBd_WD3IWS$I??ZP0kn>R-?+NAJ-S z>qgGOHjdfcbUMX>(16QfWvlaAi8yC(lN8(aGfgjvMowuSWDm^&nui8|zfA-n-j%nJ zt-G&_Mh|-E&cZj+8`LAyJZ5reB(z%Jtql77W~+pmi?9xIk8bE&huvnMBb;MpD394du@yg>Jf#~+9#*xqOyb2;|cF;~?XVw)oMi3jz+)H}+`uJvrTw3EBbJu(fvax}AD zL&MlRn%;S3^;wXcN5m~>ezMPan~8W6Uk1|BU$$y_gvf_aEiNJ7ZRIE!eK| zhux}!-~4pn<6dk0e(fawMi`67Lxop-&{FQF8s5olT}r91}JAm<+@bi$_Xz%8#^%bL!8-8Y$EeSM%}0pecW&Nev

OB}#(hK&ZWj3s`n{7HrqjtxUSa`@>~z?#eOju(Nt1z7pe-=|v*U=Ekv- zvB>rJh)W)ALCqQR(daQ*!uj&;)>8ua32{)Bh?V|;Di~W#_Yw|bx5d{m*`?_=2jD2t z`SJ?v`+8pBDQ?^7(F}h|ql35_SF4RIz}n1;bo;S<2~S||*ug7PS2x`L^I#c}?Z>#} zkSMmzuWX2<9sEv{}ruJgU^VCR&2SE z;p;lUs6OB)Gwv+O-1kV$020>bKYQ^r5v(bd@qa0TKFB_9WdibISQ%MxH#Y+NH_0lf z87r3bDWM+hLjz<&dPC{bB^Sq+V7J|n{q%zpNx#PP3mNbO#ujUNGIyeTC$xjji_Kao zqXX@VI?Y5sW7@VO1SCZ)q@ieOFrv1FM|T@ax$wfWqUvmY(rDP+c%|_wcc*3?2C;HJ zOvUW$%=Az$rRhJ0;)x$Qa|P`_b|~ZVLn@PVZg~Ja@zUt~@Hrt+RVeN{?3qY|c<}wW zVgQ5s2%?T+!!Ks^$+d2jlaDEK6NEM#RoyP}0TfG&AXR|NWlwV3;&iY9%kr#~*QZ*Y z7#M8tfocarx&cDLiYb4S_=!la4Jsc!r6E40jeF>DpatoJD74|Vf-C92&g4SL=ci}^ zuLgcq+0@K)lslTv@O%6mBPO^adFvk1#1m~ebhsIL>X=p$J(dg9lm=I< z3G$e>0KU*b{&fIqLX@>h3)STrmLXNHuv(L?*Qp}kp7IKagPGVk2IybtvF_WOf;5&m zm5c|(KZkLKb0*j*!S4$YE3AVP+xCB_0y;VZ@g_)fBYbmd?Co^Mo@tt`Hj;V8zQf+y z!i}$F$ne(&S4NRQbB=QKZJ%k-@==ArfMkeMgtbX@j%#6pM*fI6zbrK�+w#YfYu9x==Sog@SmS=U`POT9?AfMG-ESqPBdf{8;3K+yCxUZ< zbK8c_uOYeCk0Dc3F7j(q+P!hfz)&tw=kbDx%ebgrlnO%y)4YJXsQK+1#i^}44Pzcb zA?;m+wO#NMh*^iY8Wg}%(bPi~y?d$^p7j0AHM)UUCb67(X#xuuxeBpGn|9}`guLaN z)Y1#z_m`kJNIxM&93Z>i1*!NTx&FcY$d|4rjlwf0>5rt*F4;i!Yx4yaGNK-qaA?)8mN| zME*?9{c)GPk{$sul_jd=)!g>wlHILk-PYBpMs~)hvwRaNaJ;5Ae}9X+G%M+deHChJo#HOHA3&mN7)~BBMP~Ceh%QKfprNg;N z$fpAGSW-ANT#=nwHZF$?x)mCRgGd``l_pWOt~?0~4ha<|k7h^w03S{9yvkGY7I*=c&@fz&@Sl@2aZQ_LRhM?C9@@)TwUKZQ%hq|y}Ac|fp1 zt}^PxvpFZ3Nz5?K?=g!7??3VFZudzh6NGVJS}57zTgvH=Yi?)mkfCL~w+LCI$#0ad zp+Ie76R?A7GpdQ_S}J4lSO)0S=w$ADFLo<3f-7+xVj8w^^=hOS41?NL9Df85e@dq9 zWJ3+b(8w4h58w7tcv#3rr^r2dlNpc*MSC9%zKNH4HyH8CULhA68(TaNu9Gqh|4BV= z44?cdaVr=%9AEmk3|ygV zXtj=LVmo(e^&7govvs3HX!Vg^$kI`gFS?7s`!0&u?W{c5%0Gs+~&!W@mLl~#>Tj)JHXIc*8H8umI0xMOeP_%FT>jOWN|Evn*g z<2d_^wxq$(cFZ=prmRp-a5At&1}Aq?9JjeTk%M1)DojHDI1v6DOq=|Bxa`yhIW$~O z5>$%tPpV`erbUJAK%dv$2EQGZ4wjVD#(@eJ9yCMzcCvh1=qk-jwn=QLU561ZaY*|~ z0jEzG;C;F3d`2bmA`;$@$=@uEK>cXSOy;H&A>(YV=E=yDO6DHFm=y~#zV+V9O02Qf z%=J^8U~FPs84=EI7{js<+iYLWBQD_G-up{;V7;(Ix3Vy!nhH7YB_EEY3aDqgsmf{S zV^_es=F8hg`~y}}X;@@irYz(h0T{QeN(B>y#RUvsX(2oWd%V6jo?^tSF;h6utvqru zv0EU*HUD)5KnZ|W2oLqW>*!yx>JOeQ&;yFLA9cY1FOne;+~${@{dWu{ z+W$5k;VEb()@^(ai~hWf=#M_RJ<7;Eg+Dp=N=QeCr!8#lO2G9=Ln)UzRhB@GMZzLF z{mM8Yume%!lYY{6;xAkf=oG+dnYxVY5)`YM0writXXOy9pm!ra_#6@y%)#;}tg$JQ zH~gOD${4Il*|Pt@s0UZJb7|=4Qquf~F)2y3q4*0r(i1Q$Ld^*nnVsnCZO}gh4K-f94z;@N@KPpW@8fpw2P&OokJyp!l_89P8{8|3j>pBR09$ zrGUJBZg~>te9jb_#|Ho0RLZLWB#)or>On9jMaIb($nK&(ylu8I$k~YR$7^XQC2>Ny zfiRo*NamGl(HKCHCnw)Fo*yE literal 0 HcmV?d00001 From 1b2fcf5a98f36dc74a236b589ad79e0dcd109587 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Fri, 31 May 2019 08:16:51 -0700 Subject: [PATCH 06/79] Add author lines --- conf/czbiohub.config | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/conf/czbiohub.config b/conf/czbiohub.config index ff1523e..1c788b9 100644 --- a/conf/czbiohub.config +++ b/conf/czbiohub.config @@ -1,12 +1,19 @@ /* * ------------------------------------------------- - * Nextflow config file for UPPMAX (milou / irma) + * Nextflow config file for Chan Zuckerberg Biohub * ------------------------------------------------- * Defines reference genomes, using iGenome paths * Imported under the default 'standard' Nextflow * profile in nextflow.config */ + //Profile config names for nf-core/configs + params { + config_profile_description = 'Chan Zuckerberg Biohub AWS Batch profile provided by nf-core/configs.' + config_profile_contact = 'Olga Botvinnik (@olgabot)' + config_profile_url = 'https://www.czbiohub.org/' + } + docker { enabled = true } From 9c0315addb194e2360c6fe76d53a9947d3866fab Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Fri, 31 May 2019 13:17:48 -0700 Subject: [PATCH 07/79] Use flat nongzipped files --- conf/czbiohub.config | 52 ++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/conf/czbiohub.config b/conf/czbiohub.config index 1c788b9..d51c6f2 100644 --- a/conf/czbiohub.config +++ b/conf/czbiohub.config @@ -47,59 +47,59 @@ params { // GENCODE GTF and fasta files genomes { 'GRCh38' { - fasta = "${params.gencode_base}/human/v29/GRCh38.p12.genome.fa.gz" - gtf = "${params.gencode_base}/human/v29/gencode.vM19.annotation.gtf.gz" + fasta = "${params.gencode_base}/human/v29/GRCh38.p12.genome.fa" + gtf = "${params.gencode_base}/human/v29/gencode.vM19.annotation.gtf" } 'GRCm38' { - fasta = "${params.gencode_base}/mouse/vM19/GRCm38.p6.genome.fa.gz" - gtf = "${params.gencode_base}/mouse/vM19/gencode.vM19.annotation.gtf.gz" + fasta = "${params.gencode_base}/mouse/vM19/GRCm38.p6.genome.fa" + gtf = "${params.gencode_base}/mouse/vM19/gencode.vM19.annotation.gtf" } } transgenes { 'ChR2' { - fasta = "${params.transgenes_base}/ChR2/ChR2.fa.gz" - gtf = "${params.transgenes_base}/ChR2/ChR2.gtf.gz" + fasta = "${params.transgenes_base}/ChR2/ChR2.fa" + gtf = "${params.transgenes_base}/ChR2/ChR2.gtf" } 'Cre' { - fasta = "${params.transgenes_base}/Cre/Cre.fa.gz" - gtf = "${params.transgenes_base}/Cre/Cre.gtf.gz" + fasta = "${params.transgenes_base}/Cre/Cre.fa" + gtf = "${params.transgenes_base}/Cre/Cre.gtf" } 'ERCC' { - fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa.gz" - gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf.gz" + fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa" + gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf" } 'GCaMP6m' { - fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa.gz" - gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf.gz" + fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa" + gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf" } 'GFP' { - fasta = "${params.transgenes_base}/Gfp/Gfp.fa.gz" - gtf = "${params.transgenes_base}/Gfp/Gfp.gtf.gz" + fasta = "${params.transgenes_base}/Gfp/Gfp.fa" + gtf = "${params.transgenes_base}/Gfp/Gfp.gtf" } 'NpHR' { - fasta = "${params.transgenes_base}/NpHR/NpHR.fa.gz" - gtf = "${params.transgenes_base}/NpHR/NpHR.gtf.gz" + fasta = "${params.transgenes_base}/NpHR/NpHR.fa" + gtf = "${params.transgenes_base}/NpHR/NpHR.gtf" } 'RCaMP' { - fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa.gz" - gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf.gz" + fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa" + gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf" } 'RGECO' { - fasta = "${params.transgenes_base}/RGECO/RGECO.fa.gz" - gtf = "${params.transgenes_base}/RGECO/RGECO.gtf.gz" + fasta = "${params.transgenes_base}/RGECO/RGECO.fa" + gtf = "${params.transgenes_base}/RGECO/RGECO.gtf" } 'Tdtom' { - fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa.gz" - gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf.gz" + fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa" + gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf" } 'Car-T' { - fasta = "${params.transgenes_base}/car-t/car-t.fa.gz" - gtf = "${params.transgenes_base}/car-t/car-t.gtf.gz" + fasta = "${params.transgenes_base}/car-t/car-t.fa" + gtf = "${params.transgenes_base}/car-t/car-t.gtf" } 'zsGreen' { - fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa.gz" - gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf.gz" + fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa" + gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf" } } } From 4491907f1461185b8be7e6906e9d7b583dc91fd3 Mon Sep 17 00:00:00 2001 From: Tobias Schraink Date: Tue, 4 Jun 2019 12:56:11 -0400 Subject: [PATCH 08/79] updated singularity version in conf/prince.config to reflect version installed on prince The prince admins changed the singularity version to 3.2.1 from 3.1.0, making the old version unavailable. To keep this config functional, I updated the version number for singularity. --- conf/prince.config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conf/prince.config b/conf/prince.config index 6ae4ee8..1c2ea2c 100644 --- a/conf/prince.config +++ b/conf/prince.config @@ -1,4 +1,6 @@ singularityDir = "$SCRATCH/singularity_images_nextflow" +singularityModule = "singularity/3.2.1" +squashfsModule = "squashfs/4.3" params { config_profile_description = """ @@ -17,8 +19,8 @@ singularity { process { beforeScript = """ - module load singularity/3.1.0 - module load squashfs/4.3 + module load $singularityModule + module load $squashfsModule """ .stripIndent() executor = 'slurm' From 3a481402b82debaee410afeb2b64d7ad5afd1039 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 5 Jun 2019 14:42:07 -0700 Subject: [PATCH 09/79] Get czbiohub config to work --- conf/{czbiohub.config => czbiohub_aws.config} | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) rename conf/{czbiohub.config => czbiohub_aws.config} (95%) diff --git a/conf/czbiohub.config b/conf/czbiohub_aws.config similarity index 95% rename from conf/czbiohub.config rename to conf/czbiohub_aws.config index d51c6f2..baf4550 100644 --- a/conf/czbiohub.config +++ b/conf/czbiohub_aws.config @@ -20,8 +20,15 @@ docker { process { executor = 'awsbatch' + queue = 'nextflow' } +workDir = "s3://czb-nextflow/work" + +aws.region = 'us-west-2' +executor.awscli = '/home/ec2-user/miniconda/bin/aws' +params.tracedir = './' + params { saveReference = true @@ -42,7 +49,6 @@ params { // AWS configurations awsregion = "us-west-2" awsqueue = "nextflow" - workDir = "s3://czb-nextflow/rnaseq/" // GENCODE GTF and fasta files genomes { From 268e411d10eefff821625e0f72346d52cf30842e Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 5 Jun 2019 14:42:15 -0700 Subject: [PATCH 10/79] Add czbiohub_aws to list --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 7374fa0..0ca7fbe 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -16,6 +16,7 @@ profiles { ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } + czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From ad7e26ea69388a906397c41da9363a91c76c2a1c Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 5 Jun 2019 14:42:24 -0700 Subject: [PATCH 11/79] Write out docs for czbiohub" git push --- docs/czbiohub.md | 141 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 docs/czbiohub.md diff --git a/docs/czbiohub.md b/docs/czbiohub.md new file mode 100644 index 0000000..6234fc2 --- /dev/null +++ b/docs/czbiohub.md @@ -0,0 +1,141 @@ +# nf-core/configs: CZ Biohub Configuration + +All nf-core pipelines have been successfully configured for use on the AWS Batch at the Chan Zuckerberg Biohub here. + +To use, run the pipeline with `-profile czbiohub_aws`. This will download and launch the [`czbiohub_aws.config`](../conf/czbiohub_aws.config) which has been pre-configured with a setup suitable for the AWS Batch. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Ask Olga (olga.botvinnik@czbiohub.org) if you have any questions! + +## Run the pipeline from a small AWS EC2 Instance + +The pipeline will monitor and submit jobs to AWS Batch on your behalf. To ensure that the pipeline is successful, it will need to be run from a computer that has constant internet connection. Unfortunately for us, Biohub has spotty WiFi and even for short pipelines, it is highly recommended to run them from AWS. Make sure you have [aegea](https://github.com/czbiohub/codonboarding/blob/master/guides/aegea.md) installed to make launching AWS instances from the command line much easier. + +### 1. Launch the instance + +There is an Elastic Compute Cluster (EC2) Amazon machine image (AMI) set up with everything you need for Nextflow already installed. Its ID is `ami-091ec599f8e77734d` and can be launched from the command line with aegea like this: + +``` +aegea launch --iam-role S3fromEC2 -t t2.small --ami ami-091ec599f8e77734d --subnet subnet-672e832e $USER-nextflow +``` + +For example: + +``` +aegea launch --iam-role S3fromEC2 -t t2.small --ami ami-0adcc973d6f458a1e --subnet subnet-672e832e olgabot-nextflow +``` + +### 2. Log into the instance + +Log into the instance with `aegea ssh`: + +``` +aegea ssh ec2-user@$USER-nextflow +``` + +For a concrete example: + +``` +aegea ssh ec2-user@olgabot-nextflow +``` + +### 3. Start tmux + +[tmux](https://hackernoon.com/a-gentle-introduction-to-tmux-8d784c404340) is a "Terminal Multiplexer" that allows for commands to continue running even when you have closed your laptop and lost your connection. Start a new tmux session with `tmux new` + +``` +tmux new +``` + +Now you can run pipelines with abandon! + +### 4. Make a GitHub repo for your workflows (optional :) + +To make sharing your pipelines and commands easy between your teammates, it's best to share code in a GitHub repository. One way is to store the commands in a Makefile ([example](https://github.com/czbiohub/kh-workflows/blob/master/nf-kmer-similarity/Makefile)) which can contain multiple `nextflow run` commands so that you don't need to remember the S3 bucket or output directory for every single one. [Makefiles](https://kbroman.org/minimal_make/) are broadly used in the software community for running many complex commands. Makefiles can have a lot of dependencies and be confusing, so we're only going to write *simple* Makefiles. + +``` +rnaseq: + nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ + +human_mouse_zebrafish: + nextflow run czbiohub/nf-kmer-similarity -latest -profile aws \ + --samples s3://kmer-hashing/hematopoeisis/smartseq2/human_mouse_zebrafish/samples.csv + + +merkin2012_aws: + nextflow run czbiohub/nf-kmer-similarity -latest --sra "SRP016501" \ + -r olgabot/support-csv-directory-or-sra \ + -profile aws +``` + +In this example, one would run the `rnaseq` rule and the nextflow command beneath it with: + +``` +make rnaseq +``` + +If one wanted to run a different command, e.g. `human_mouse_zebrafish`, they would specify that command instead. For example: + +``` +make human_mouse_zebrafish +``` + +Makefiles are a very useful way of storing longer commands with short mnemonic words. + + +Once you [create a new repository](https://github.com/organizations/czbiohub/repositories/new) (best to initialize with a `.gitignore`, license - MIT and `README`), clone that repository to your EC2 instance. For example, if the repository is called `kh-workflows`, this is what the command would look like: + +``` +git clone https://github.com/czbiohub/kh-workflows +``` + +Now both create and edit a `Makefile`: + +``` +cd +nano Makefile +``` + +Write your rule with a colon after it, and on the next line must be a **tab**, not spaces. Once you're done, exit the program (the `^` command shown in nano means "Control"), write the file, add it to git, commit it, and push it up to GitHub. + + +``` +git add Makefile +git commit -m "Added makefile" +git push origin master +``` + + +### 5. Run your workflow!! + +Remember to specify `-profile czbiohub_aws` to grab the CZ Biohub-specific AWS configurations, and an `--outdir` with an AWS S3 bucket so you don't run out of space on your small AMI + +``` +nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ +``` + +### 6. If you lose connection, how do you re-attach the tmux session? + +If you close your laptop, get onto the train, or lose WiFi connection, you may lose connection to your AWS EC2 instance. To reattach, use the command `tmux attach` and you should see your Nextflow output! + +``` +tmux attach +``` + + +## iGenomes specific configuration + +A local copy of the iGenomes resource has been made available on `s3://czbiohub-reference` (in `us-west-2` region) so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline. +You can do this by simply using the `--genome ` parameter. + +For Human and Mouse, we use [GENCODE](https://www.gencodegenes.org/) gene annotations. This doesn't change how you would specify the genome name, only that the pipelines run with the `czbiohub_aws` profile would be with GENCODE rather than iGenomes. + + +>NB: You will need an account to use the HPC cluster on PROFILE CLUSTER in order to run the pipeline. If in doubt contact IT. + +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From f40ed194e3afb9d0c6ad0c511943c404ad690cd9 Mon Sep 17 00:00:00 2001 From: Szilveszter Juhos Date: Mon, 24 Jun 2019 09:54:37 +0200 Subject: [PATCH 12/79] Using correct reference location for munin --- conf/munin.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/munin.config b/conf/munin.config index 5d95f01..bca56f7 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -28,5 +28,5 @@ params { max_cpus = 16 max_time = 72.h // illumina iGenomes reference file paths on UPPMAX - igenomes_base = '/data0/btb/references/igenomes/' + igenomes_base = '/data1/references/igenomes/' } From acad9ac7e5eb850361a4797000199641ef29b3e9 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 24 Jun 2019 17:50:36 -0700 Subject: [PATCH 13/79] Try to clean up czb update --- conf/czbiohub_aws.config | 123 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 conf/czbiohub_aws.config diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config new file mode 100644 index 0000000..5c47c12 --- /dev/null +++ b/conf/czbiohub_aws.config @@ -0,0 +1,123 @@ +/* + * ------------------------------------------------- + * Nextflow config file for Chan Zuckerberg Biohub + * ------------------------------------------------- + * Defines reference genomes, using iGenome paths + * Imported under the default 'standard' Nextflow + * profile in nextflow.config + */ + + //Profile config names for nf-core/configs + params { + config_profile_description = 'Chan Zuckerberg Biohub AWS Batch profile provided by nf-core/configs.' + config_profile_contact = 'Olga Botvinnik (@olgabot)' + config_profile_url = 'https://www.czbiohub.org/' + } + +docker { + enabled = true +} + +process { + executor = 'awsbatch' + queue = 'nextflow' + errorStrategy = 'ignore' +} + +workDir = "s3://czb-nextflow/intermediates/" + +aws.region = 'us-west-2' +executor.awscli = '/home/ec2-user/miniconda/bin/aws' +params.tracedir = './' + +params { + saveReference = true + + // Largest SPOT instances available on AWS: https://ec2instances.info/ + max_memory = 1952.GB + max_cpus = 96 + max_time = 240.h + + seq_center = "czbiohub" + + // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + igenomes_base = "s3://czbiohub-reference/igenomes" + + // GENCODE (human + mouse) reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + gencode_base = "s3://czbiohub-reference/gencode" + transgenes_base = "s3://czbiohub-reference/transgenes" + + // AWS configurations + awsregion = "us-west-2" + awsqueue = "nextflow" + + igenomesIgnore = true + + fc_extra_attributes = 'gene_name' + fc_group_features = 'gene_id' + fc_group_features_type = 'gene_type' + + // GENCODE GTF and fasta files + genomes { + 'GRCh38' { + fasta = "${params.gencode_base}/human/v30/GRCh38.p12.genome.ERCC92.fa" + gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gene_type.gtf" + transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" + star = "${params.gencode_base}/human/v30/STARIndex/" + } + 'GRCm38' { + fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" + gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gene_type.gtf" + transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" + } + } + + transgenes { + 'ChR2' { + fasta = "${params.transgenes_base}/ChR2/ChR2.fa" + gtf = "${params.transgenes_base}/ChR2/ChR2.gtf" + } + 'Cre' { + fasta = "${params.transgenes_base}/Cre/Cre.fa" + gtf = "${params.transgenes_base}/Cre/Cre.gtf" + } + 'ERCC' { + fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa" + gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf" + } + 'GCaMP6m' { + fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa" + gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf" + } + 'GFP' { + fasta = "${params.transgenes_base}/Gfp/Gfp.fa" + gtf = "${params.transgenes_base}/Gfp/Gfp.gtf" + } + 'NpHR' { + fasta = "${params.transgenes_base}/NpHR/NpHR.fa" + gtf = "${params.transgenes_base}/NpHR/NpHR.gtf" + } + 'RCaMP' { + fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa" + gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf" + } + 'RGECO' { + fasta = "${params.transgenes_base}/RGECO/RGECO.fa" + gtf = "${params.transgenes_base}/RGECO/RGECO.gtf" + } + 'Tdtom' { + fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa" + gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf" + } + 'Car-T' { + fasta = "${params.transgenes_base}/car-t/car-t.fa" + gtf = "${params.transgenes_base}/car-t/car-t.gtf" + } + 'zsGreen' { + fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa" + gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf" + } + } +} From b0c67e0a9b45112c570b0a9856a4ed479190ab13 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 24 Jun 2019 17:52:10 -0700 Subject: [PATCH 14/79] Add both seqCenter and seq_center --- conf/czbiohub_aws.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index 5c47c12..af7707a 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -38,7 +38,9 @@ params { max_cpus = 96 max_time = 240.h + // Compatible with multiple versions of rnaseq pipeline seq_center = "czbiohub" + seqCenter = "czbiohub" // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket // No final slash because it's added later From b82d68c009a4deb4c193bd440982381c4ba8556d Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 24 Jun 2019 17:55:01 -0700 Subject: [PATCH 15/79] Add docs --- docs/czbiohub.md | 125 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 docs/czbiohub.md diff --git a/docs/czbiohub.md b/docs/czbiohub.md new file mode 100644 index 0000000..5e98f4f --- /dev/null +++ b/docs/czbiohub.md @@ -0,0 +1,125 @@ +# nf-core/configs: CZ Biohub Configuration + +All nf-core pipelines have been successfully configured for use on the AWS Batch at the Chan Zuckerberg Biohub here. + +To use, run the pipeline with `-profile czbiohub_aws`. This will download and launch the [`czbiohub_aws.config`](../conf/czbiohub_aws.config) which has been pre-configured with a setup suitable for the AWS Batch. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Ask Olga (olga.botvinnik@czbiohub.org) if you have any questions! + +## Run the pipeline from a small AWS EC2 Instance + +The pipeline will monitor and submit jobs to AWS Batch on your behalf. To ensure that the pipeline is successful, it will need to be run from a computer that has constant internet connection. Unfortunately for us, Biohub has spotty WiFi and even for short pipelines, it is highly recommended to run them from AWS. + +### 1. Start tmux + +[tmux](https://hackernoon.com/a-gentle-introduction-to-tmux-8d784c404340) is a "Terminal Multiplexer" that allows for commands to continue running even when you have closed your laptop. Start a new tmux session with `tmux new` and we'll name this session `nextflow`. + +``` +tmux new -n nextflow +``` + +Now you can run pipelines with abandon! + +### 2. Make a GitHub repo for your workflows (optional :) + +To make sharing your pipelines and commands easy between your teammates, it's best to share code in a GitHub repository. One way is to store the commands in a Makefile ([example](https://github.com/czbiohub/kh-workflows/blob/master/nf-kmer-similarity/Makefile)) which can contain multiple `nextflow run` commands so that you don't need to remember the S3 bucket or output directory for every single one. [Makefiles](https://kbroman.org/minimal_make/) are broadly used in the software community for running many complex commands. Makefiles can have a lot of dependencies and be confusing, so we're only going to write *simple* Makefiles. + +``` +rnaseq: + nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ + +human_mouse_zebrafish: + nextflow run czbiohub/nf-kmer-similarity -latest -profile aws \ + --samples s3://kmer-hashing/hematopoeisis/smartseq2/human_mouse_zebrafish/samples.csv + + +merkin2012_aws: + nextflow run czbiohub/nf-kmer-similarity -latest --sra "SRP016501" \ + -r olgabot/support-csv-directory-or-sra \ + -profile aws +``` + +In this example, one would run the `rnaseq` rule and the nextflow command beneath it with: + +``` +make rnaseq +``` + +If one wanted to run a different command, e.g. `human_mouse_zebrafish`, they would specify that command instead. For example: + +``` +make human_mouse_zebrafish +``` + +Makefiles are a very useful way of storing longer commands with short mnemonic words. + + +Once you [create a new repository](https://github.com/organizations/czbiohub/repositories/new) (best to initialize with a `.gitignore`, license - MIT and `README`), clone that repository to your EC2 instance. For example, if the repository is called `kh-workflows`, this is what the command would look like: + +``` +git clone https://github.com/czbiohub/kh-workflows +``` + +Now both create and edit a `Makefile`: + +``` +cd +nano Makefile +``` + +Write your rule with a colon after it, and on the next line must be a **tab**, not spaces. Once you're done, exit the program (the `^` command shown in nano means "Control"), write the file, add it to git, commit it, and push it up to GitHub. + + +``` +git add Makefile +git commit -m "Added makefile" +git push origin master +``` + + +### 3. Run your workflow!! + +Remember to specify `-profile czbiohub_aws` to grab the CZ Biohub-specific AWS configurations, and an `--outdir` with an AWS S3 bucket so you don't run out of space on your small AMI + +``` +nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ +``` + +### 4. If you lose connection, how do you restart the jobs? + +If you close your laptop, get onto the train, or lose WiFi connection, you may lose connection to AWS and may need to restart the jobs. To reattach, use the command `tmux attach` and you should see your Nextflow output! To get the named session, use: + +``` +tmux attach -n nextflow +``` + +To restart the jobs from where you left off, add the `-resume` flag to your `nextflow` command: + + +``` +nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ \ + -resume +``` + +It's important that this command be re-run from the same directory as there is a "hidden" `.nextflow` folder that contains all the metadata and information about previous runs. + +## iGenomes specific configuration + +A local copy of the iGenomes resource has been made available on `s3://czbiohub-reference/igenomes` (in `us-west-2` region) so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline. +You can do this by simply using the `--genome ` parameter. + +For Human and Mouse, we use [GENCODE](https://www.gencodegenes.org/) gene annotations. This doesn't change how you would specify the genome name, only that the pipelines run with the `czbiohub_aws` profile would be with GENCODE rather than iGenomes. + + +>NB: You will need an account to use the HPC cluster on PROFILE CLUSTER in order to run the pipeline. If in doubt contact IT. + +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From cfcb533b959d5fb2c4deb5750c6bb5645d5db009 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Tue, 25 Jun 2019 19:02:49 -0700 Subject: [PATCH 16/79] Add czbiohub_aws to list of configs" --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index dfcb4d2..0bfc474 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -16,6 +16,7 @@ profiles { ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } + czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From d1983f8acd15354160e60458355eee2cf75f9b56 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 26 Jun 2019 17:56:26 -0700 Subject: [PATCH 17/79] Use new and improved GENCODE+ERCC annotation --- conf/czbiohub_aws.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index af7707a..a89588b 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -65,13 +65,13 @@ params { genomes { 'GRCh38' { fasta = "${params.gencode_base}/human/v30/GRCh38.p12.genome.ERCC92.fa" - gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gene_type.gtf" + gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gtf" transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" star = "${params.gencode_base}/human/v30/STARIndex/" } 'GRCm38' { fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" - gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gene_type.gtf" + gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gtf" transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" } } From 70b5205d153f4313bdf586837d0eb81e8b5f2607 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Thu, 27 Jun 2019 15:02:47 -0700 Subject: [PATCH 18/79] Add human salmon index and mouse star index --- conf/czbiohub_aws.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index a89588b..bd0228d 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -68,11 +68,13 @@ params { gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gtf" transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" star = "${params.gencode_base}/human/v30/STARIndex/" + salmon_index = "${params.gencode_base}/human/v30/salmon_index/" } 'GRCm38' { fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gtf" transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" + start = "${params.gencode_base}/mouse/vM21/STARIndex/" } } From 6a2d2003f0fed491ccabc130c36e6301616983fa Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 1 Jul 2019 09:25:46 -0700 Subject: [PATCH 19/79] Add high priority AWS batch queue to CZBiohub AWS --- conf/czbiohub_aws.config | 2 +- conf/czbiohub_aws_highpriority.config | 127 ++++++++++++++++++++++++++ nfcore_custom.config | 1 + 3 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 conf/czbiohub_aws_highpriority.config diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index bd0228d..d20b405 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -20,7 +20,7 @@ docker { process { executor = 'awsbatch' - queue = 'nextflow' + queue = 'default-971039e0-830c-11e9-9e0b-02c5b84a8036' errorStrategy = 'ignore' } diff --git a/conf/czbiohub_aws_highpriority.config b/conf/czbiohub_aws_highpriority.config new file mode 100644 index 0000000..2bf9a4d --- /dev/null +++ b/conf/czbiohub_aws_highpriority.config @@ -0,0 +1,127 @@ +/* + * ------------------------------------------------- + * Nextflow config file for Chan Zuckerberg Biohub + * ------------------------------------------------- + * Defines reference genomes, using iGenome paths + * Imported under the default 'standard' Nextflow + * profile in nextflow.config + */ + + //Profile config names for nf-core/configs + params { + config_profile_description = 'Chan Zuckerberg Biohub AWS Batch profile provided by nf-core/configs.' + config_profile_contact = 'Olga Botvinnik (@olgabot)' + config_profile_url = 'https://www.czbiohub.org/' + } + +docker { + enabled = true +} + +process { + executor = 'awsbatch' + queue = 'highpriority-971039e0-830c-11e9-9e0b-02c5b84a8036' + errorStrategy = 'ignore' +} + +workDir = "s3://czb-nextflow/intermediates/" + +aws.region = 'us-west-2' +executor.awscli = '/home/ec2-user/miniconda/bin/aws' +params.tracedir = './' + +params { + saveReference = true + + // Largest SPOT instances available on AWS: https://ec2instances.info/ + max_memory = 1952.GB + max_cpus = 96 + max_time = 240.h + + // Compatible with multiple versions of rnaseq pipeline + seq_center = "czbiohub" + seqCenter = "czbiohub" + + // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + igenomes_base = "s3://czbiohub-reference/igenomes" + + // GENCODE (human + mouse) reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + gencode_base = "s3://czbiohub-reference/gencode" + transgenes_base = "s3://czbiohub-reference/transgenes" + + // AWS configurations + awsregion = "us-west-2" + awsqueue = "nextflow" + + igenomesIgnore = true + + fc_extra_attributes = 'gene_name' + fc_group_features = 'gene_id' + fc_group_features_type = 'gene_type' + + // GENCODE GTF and fasta files + genomes { + 'GRCh38' { + fasta = "${params.gencode_base}/human/v30/GRCh38.p12.genome.ERCC92.fa" + gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gtf" + transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" + star = "${params.gencode_base}/human/v30/STARIndex/" + salmon_index = "${params.gencode_base}/human/v30/salmon_index/" + } + 'GRCm38' { + fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" + gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gtf" + transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" + start = "${params.gencode_base}/mouse/vM21/STARIndex/" + } + } + + transgenes { + 'ChR2' { + fasta = "${params.transgenes_base}/ChR2/ChR2.fa" + gtf = "${params.transgenes_base}/ChR2/ChR2.gtf" + } + 'Cre' { + fasta = "${params.transgenes_base}/Cre/Cre.fa" + gtf = "${params.transgenes_base}/Cre/Cre.gtf" + } + 'ERCC' { + fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa" + gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf" + } + 'GCaMP6m' { + fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa" + gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf" + } + 'GFP' { + fasta = "${params.transgenes_base}/Gfp/Gfp.fa" + gtf = "${params.transgenes_base}/Gfp/Gfp.gtf" + } + 'NpHR' { + fasta = "${params.transgenes_base}/NpHR/NpHR.fa" + gtf = "${params.transgenes_base}/NpHR/NpHR.gtf" + } + 'RCaMP' { + fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa" + gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf" + } + 'RGECO' { + fasta = "${params.transgenes_base}/RGECO/RGECO.fa" + gtf = "${params.transgenes_base}/RGECO/RGECO.gtf" + } + 'Tdtom' { + fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa" + gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf" + } + 'Car-T' { + fasta = "${params.transgenes_base}/car-t/car-t.fa" + gtf = "${params.transgenes_base}/car-t/car-t.gtf" + } + 'zsGreen' { + fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa" + gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf" + } + } +} diff --git a/nfcore_custom.config b/nfcore_custom.config index 0bfc474..de5961c 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -17,6 +17,7 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } + czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From 45d36a150661b4f33a3847d345f17d41b74fa8ee Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 3 Jul 2019 09:21:16 +0200 Subject: [PATCH 20/79] Update cfc.config Add Weblog for CFC cluster --- conf/cfc.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/cfc.config b/conf/cfc.config index dd69647..2f9047d 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -14,6 +14,11 @@ process { executor = 'slurm' } +weblog{ + enabled = true + url = 'http://services.qbic.uni-tuebingen.de:8080/workflows' +} + params { igenomes_base = '/nfsmounts/igenomes' max_memory = 60.GB From 917ec0863d5e427940ec8a19ecdc258df786aef1 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 3 Jul 2019 16:01:03 -0700 Subject: [PATCH 21/79] add default trimming --- conf/czbiohub_aws.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index d20b405..71d1a9b 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -61,6 +61,8 @@ params { fc_group_features = 'gene_id' fc_group_features_type = 'gene_type' + trim_pattern = '_+S\\d+' + // GENCODE GTF and fasta files genomes { 'GRCh38' { From 069963ff8a5ce9f6cad3a7be71e44782f3417ee4 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 3 Jul 2019 16:01:47 -0700 Subject: [PATCH 22/79] Make czbiohub_aws_highpriority only set the AWS Batch queue and nothing else --- conf/czbiohub_aws_highpriority.config | 115 -------------------------- nfcore_custom.config | 5 +- 2 files changed, 4 insertions(+), 116 deletions(-) diff --git a/conf/czbiohub_aws_highpriority.config b/conf/czbiohub_aws_highpriority.config index 2bf9a4d..5ab796a 100644 --- a/conf/czbiohub_aws_highpriority.config +++ b/conf/czbiohub_aws_highpriority.config @@ -7,121 +7,6 @@ * profile in nextflow.config */ - //Profile config names for nf-core/configs - params { - config_profile_description = 'Chan Zuckerberg Biohub AWS Batch profile provided by nf-core/configs.' - config_profile_contact = 'Olga Botvinnik (@olgabot)' - config_profile_url = 'https://www.czbiohub.org/' - } - -docker { - enabled = true -} - process { - executor = 'awsbatch' queue = 'highpriority-971039e0-830c-11e9-9e0b-02c5b84a8036' - errorStrategy = 'ignore' -} - -workDir = "s3://czb-nextflow/intermediates/" - -aws.region = 'us-west-2' -executor.awscli = '/home/ec2-user/miniconda/bin/aws' -params.tracedir = './' - -params { - saveReference = true - - // Largest SPOT instances available on AWS: https://ec2instances.info/ - max_memory = 1952.GB - max_cpus = 96 - max_time = 240.h - - // Compatible with multiple versions of rnaseq pipeline - seq_center = "czbiohub" - seqCenter = "czbiohub" - - // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket - // No final slash because it's added later - igenomes_base = "s3://czbiohub-reference/igenomes" - - // GENCODE (human + mouse) reference file paths on CZ Biohub reference s3 bucket - // No final slash because it's added later - gencode_base = "s3://czbiohub-reference/gencode" - transgenes_base = "s3://czbiohub-reference/transgenes" - - // AWS configurations - awsregion = "us-west-2" - awsqueue = "nextflow" - - igenomesIgnore = true - - fc_extra_attributes = 'gene_name' - fc_group_features = 'gene_id' - fc_group_features_type = 'gene_type' - - // GENCODE GTF and fasta files - genomes { - 'GRCh38' { - fasta = "${params.gencode_base}/human/v30/GRCh38.p12.genome.ERCC92.fa" - gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gtf" - transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" - star = "${params.gencode_base}/human/v30/STARIndex/" - salmon_index = "${params.gencode_base}/human/v30/salmon_index/" - } - 'GRCm38' { - fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" - gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gtf" - transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" - start = "${params.gencode_base}/mouse/vM21/STARIndex/" - } - } - - transgenes { - 'ChR2' { - fasta = "${params.transgenes_base}/ChR2/ChR2.fa" - gtf = "${params.transgenes_base}/ChR2/ChR2.gtf" - } - 'Cre' { - fasta = "${params.transgenes_base}/Cre/Cre.fa" - gtf = "${params.transgenes_base}/Cre/Cre.gtf" - } - 'ERCC' { - fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa" - gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf" - } - 'GCaMP6m' { - fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa" - gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf" - } - 'GFP' { - fasta = "${params.transgenes_base}/Gfp/Gfp.fa" - gtf = "${params.transgenes_base}/Gfp/Gfp.gtf" - } - 'NpHR' { - fasta = "${params.transgenes_base}/NpHR/NpHR.fa" - gtf = "${params.transgenes_base}/NpHR/NpHR.gtf" - } - 'RCaMP' { - fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa" - gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf" - } - 'RGECO' { - fasta = "${params.transgenes_base}/RGECO/RGECO.fa" - gtf = "${params.transgenes_base}/RGECO/RGECO.gtf" - } - 'Tdtom' { - fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa" - gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf" - } - 'Car-T' { - fasta = "${params.transgenes_base}/car-t/car-t.fa" - gtf = "${params.transgenes_base}/car-t/car-t.gtf" - } - 'zsGreen' { - fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa" - gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf" - } - } } diff --git a/nfcore_custom.config b/nfcore_custom.config index de5961c..72dca30 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -17,7 +17,10 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } - czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } + czbiohub_aws_highpriority { + includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" + includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" + } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From cb507177d5ceca20722bea2acda201c040d9300b Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Fri, 5 Jul 2019 08:03:37 +0200 Subject: [PATCH 23/79] Updating CCGA config file --- conf/ccga.config | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/conf/ccga.config b/conf/ccga.config index f29db9c..b9130e0 100644 --- a/conf/ccga.config +++ b/conf/ccga.config @@ -2,15 +2,20 @@ params { config_profile_description = 'CCGA cluster profile provided by nf-core/configs.' config_profile_contact = 'Marc Hoeppner (@marchoeppner)' - config_profile_url = 'https://www.ikmb.uni-kiel.de/' + config_profile_url = 'https://www.ccga.uni-kiel.de/' } /* * ------------------------------------------------- - * Nextflow config file with environment modules for RZCluster in Kiel + * Nextflow config file for CCGA cluster in Kiel * ------------------------------------------------- */ +singularity { + enabled = true + runOptions = "-B /ifs -B /scratch -B /work_beegfs" +} + executor { queueSize=100 } @@ -29,4 +34,7 @@ params { // illumina iGenomes reference file paths on RZCluster igenomes_base = '/ifs/data/nfs_share/ikmb_repository/references/iGenomes/references/' saveReference = true + max_memory = 128.GB + max_cpus = 16 + max_time = 120.h } From d66bf4e4786799301846b34dfd9602d17c4ce3ce Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Fri, 5 Jul 2019 08:28:18 +0200 Subject: [PATCH 24/79] Updating CCGA documentation --- docs/ccga.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 docs/ccga.md diff --git a/docs/ccga.md b/docs/ccga.md new file mode 100644 index 0000000..4a38bd9 --- /dev/null +++ b/docs/ccga.md @@ -0,0 +1,18 @@ +# nf-core/configs: CCGA Configuration + +Deployment and testing of nf-core pipelines at the CCGA cluster is on-going. + +To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the BINAC cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on the cluster. You can do this by issuing the commands below: + +```bash +## Load Nextflow and Singularity environment modules +module purge +module load IKMB +module load Java/1.8.0 +module load Nextflow +module load singularity3.1.0 +``` + +>NB: Access to the CCGA cluster is restricted to IKMB/CCGA employes. Please talk to Marc Hoeppner to get access (@marchoeppner). From fe0153befd41e7cd054a398d64dac7da89f1f45d Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Fri, 5 Jul 2019 08:30:02 +0200 Subject: [PATCH 25/79] Updating CCGA documentation --- docs/ccga.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ccga.md b/docs/ccga.md index 4a38bd9..1157cb3 100644 --- a/docs/ccga.md +++ b/docs/ccga.md @@ -2,7 +2,7 @@ Deployment and testing of nf-core pipelines at the CCGA cluster is on-going. -To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the BINAC cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on the cluster. You can do this by issuing the commands below: From 9106559b2f319c0f78a021bc28abd714ab957c40 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 15 Jul 2019 17:53:11 +0200 Subject: [PATCH 26/79] Update cfc.config --- conf/cfc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index 2f9047d..6fc3b26 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -16,7 +16,7 @@ process { weblog{ enabled = true - url = 'http://services.qbic.uni-tuebingen.de:8080/workflows' + url = 'http://services.qbic.uni-tuebingen.de:8080/workflowservice/workflows' } params { From 1b30c5a4d24301ad1100e305f4ad1885b9f22b3f Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Wed, 24 Jul 2019 11:32:02 +0200 Subject: [PATCH 27/79] Add the Pasteur profile --- conf/pasteur.config | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 conf/pasteur.config diff --git a/conf/pasteur.config b/conf/pasteur.config new file mode 100644 index 0000000..03971dc --- /dev/null +++ b/conf/pasteur.config @@ -0,0 +1,24 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'The Institut Pasteur HPC cluster profile' + config_profile_contact = 'Remi Planel (@rplanel)' + config_profile_url = 'https://research.pasteur.fr/en/service/tars-cluster/' +} + +singularity { + enabled = true + autoMounts = true + runOptions = '-B /local/scratch:/tmp' +} + +process { + executor = 'slurm' + clusterOptions = { "-A $params.project ${params.clusterOptions ?: ''}" } +} + +params { + igenomesIgnore = true + max_memory = 256.GB + max_cpus = 28 + max_time = 24.h +} From 1cb132a4572dc85bd7478bdb881cb1934e0cdf0f Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Wed, 24 Jul 2019 11:32:25 +0200 Subject: [PATCH 28/79] Define a profile for the Pasteur config --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 41b0ade..a9c0729 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -33,6 +33,7 @@ profiles { uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } prince { includeConfig "${params.custom_config_base}/conf/prince.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } + pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } } // If user hostnames contain one of these substring and they are From fbb33fde63700e1f76852ad7df2780bba4b7ec6a Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Wed, 24 Jul 2019 11:43:08 +0200 Subject: [PATCH 29/79] Details --- conf/pasteur.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pasteur.config b/conf/pasteur.config index 03971dc..2548d3f 100644 --- a/conf/pasteur.config +++ b/conf/pasteur.config @@ -2,7 +2,7 @@ params { config_profile_description = 'The Institut Pasteur HPC cluster profile' config_profile_contact = 'Remi Planel (@rplanel)' - config_profile_url = 'https://research.pasteur.fr/en/service/tars-cluster/' + config_profile_url = 'https://research.pasteur.fr/en/service/tars-cluster' } singularity { From 71bc98afaef852562d5664d7861cc31cb870db79 Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Mon, 29 Jul 2019 12:06:29 +0200 Subject: [PATCH 30/79] Adding config file for CCGA dx cluster --- docs/ccga_dx.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/ccga_dx.md diff --git a/docs/ccga_dx.md b/docs/ccga_dx.md new file mode 100644 index 0000000..62bf29c --- /dev/null +++ b/docs/ccga_dx.md @@ -0,0 +1,9 @@ +# nf-core/configs: CCGA DX Configuration + +Deployment and testing of nf-core pipelines at the CCGA DX cluster is on-going. + +To use, run the pipeline with `-profile ccga_dx`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to have Nextflow installed. + +>NB: Access to the CCGA DX cluster is restricted to IKMB/CCGA employes. Please talk to Marc Hoeppner to get access (@marchoeppner). From b31db18dfb0b790aecc81c3685a5cc30a57fccec Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Mon, 29 Jul 2019 12:08:07 +0200 Subject: [PATCH 31/79] Adding config file for CCGA dx cluster --- README.md | 1 + conf/ccga_dx.config | 37 +++++++++++++++++++++++++++++++++++++ nfcore_custom.config | 1 + 3 files changed, 39 insertions(+) create mode 100644 conf/ccga_dx.config diff --git a/README.md b/README.md index 73b01be..d865c89 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ Currently documentation is available for the following clusters: * [BINAC](docs/binac.md) * [BIGPURPLE](docs/bigpurple.md) * [CCGA](docs/ccga.md) +* [CCGA-DX]/docs/ccga_dx.md) * [CFC](docs/binac.md) * [CRICK](docs/crick.md) * [GIS](docs/gis.md) diff --git a/conf/ccga_dx.config b/conf/ccga_dx.config new file mode 100644 index 0000000..0af95e7 --- /dev/null +++ b/conf/ccga_dx.config @@ -0,0 +1,37 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'CCGA DX cluster profile provided by nf-core/configs.' + config_profile_contact = 'Marc Hoeppner (@marchoeppner)' + config_profile_url = 'https://www.ccga.uni-kiel.de/' +} + +/* + * ------------------------------------------------- + * Nextflow config file for CCGA cluster in Kiel + * ------------------------------------------------- + */ + +singularity { + enabled = true +} + +executor { + queueSize=100 +} + +process { + + // Global process config + executor = 'slurm' + queue = 'htc' + +} + +params { + // illumina iGenomes reference file paths on DX Cluster + igenomes_base = '/mnt/ld_ng_out/sukmb352/references/iGenomes/references/' + saveReference = true + max_memory = 250.GB + max_cpus = 20 + max_time = 240.h +} diff --git a/nfcore_custom.config b/nfcore_custom.config index 41b0ade..d4ae35b 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -14,6 +14,7 @@ params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/$ profiles { binac { includeConfig "${params.custom_config_base}/conf/binac.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } + ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } From 7d5de651aa238c77f6bee3347622d34c1aa8a942 Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Mon, 29 Jul 2019 12:45:08 +0200 Subject: [PATCH 32/79] Update README.md Co-Authored-By: Harshil Patel --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d865c89..d7b43fb 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Currently documentation is available for the following clusters: * [BINAC](docs/binac.md) * [BIGPURPLE](docs/bigpurple.md) * [CCGA](docs/ccga.md) -* [CCGA-DX]/docs/ccga_dx.md) +* [CCGA_DX](/docs/ccga_dx.md) * [CFC](docs/binac.md) * [CRICK](docs/crick.md) * [GIS](docs/gis.md) From b74f020f14d3254d67d85edaa2a3c8d20dfea389 Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Mon, 29 Jul 2019 12:45:30 +0200 Subject: [PATCH 33/79] Update docs/ccga_dx.md Co-Authored-By: Harshil Patel --- docs/ccga_dx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ccga_dx.md b/docs/ccga_dx.md index 62bf29c..0cea81e 100644 --- a/docs/ccga_dx.md +++ b/docs/ccga_dx.md @@ -2,7 +2,7 @@ Deployment and testing of nf-core pipelines at the CCGA DX cluster is on-going. -To use, run the pipeline with `-profile ccga_dx`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile ccga_dx`. This will download and launch the [`ccga_dx.config`](../conf/ccga_dx.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Before running the pipeline you will need to have Nextflow installed. From 96ea16ddbd9082f630c4836d2cb032542c15151c Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 15:14:18 +0200 Subject: [PATCH 34/79] Ignore vscode workspace --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 07c0144..8aa0735 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ work/ data/ results/ .DS_Store +*.code-workspace \ No newline at end of file From b4cffe0455d1523ad179fd0d125857511e722216 Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 15:15:07 +0200 Subject: [PATCH 35/79] Remove buggy cluster option --- conf/pasteur.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/pasteur.config b/conf/pasteur.config index 2548d3f..1d95131 100644 --- a/conf/pasteur.config +++ b/conf/pasteur.config @@ -13,7 +13,6 @@ singularity { process { executor = 'slurm' - clusterOptions = { "-A $params.project ${params.clusterOptions ?: ''}" } } params { From 6f264ac2caca9429e27e72c1347d96dfdfcd9457 Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 15:15:30 +0200 Subject: [PATCH 36/79] Add pasteur profile doc --- docs/pasteur.md | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/pasteur.md diff --git a/docs/pasteur.md b/docs/pasteur.md new file mode 100644 index 0000000..153f4d2 --- /dev/null +++ b/docs/pasteur.md @@ -0,0 +1,54 @@ +# nf-core/configs: Institut Pasteur Configuration + +All nf-core pipelines have been successfully configured for use on the tars cluster at the Institut Pasteur. + +To use, run the pipeline with `-profile pasteur`. This will download and launch the [`pasteur.config`](../conf/pasteur.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + + + +## Running the workflow on the Pasteur cluster + +>Nextflow is not installed by default on the Pasteur cluster. +- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) + + +>Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. +Nextflow shouldn't run directly on the submission node but on a compute node. +The compute nodes don't have access to internet so you need to run it offline. To do that: +1. Create a virtualenv to install nf-core in order to get the workflow +```bash +module purge +module load Python/3.6.0 +module load java +module load singularity +cd /path/to/nf-core/workflows +virtualenv .venv -p python3 +. .venv/bin/activate +``` +2. Install nf-core: [here](https://nf-co.re/tools#installation) +3. Get nf-core pipeline and container: [here](https://nf-co.re/tools#downloading-pipelines-for-offline-use) +4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/rnaseq/blob/master/docs/usage.md#--custom_config_base) +5. Run nextflow on a compute node: +```bash +# create a terminal +tmux + +# Get a compute node +salloc + +# Load the dependencies if not done before +module purge +module load java +module load singularity + +# Run nextflow workflow +nextflow run \\ +/path/to/pipeline-dir/from/step/2/workflow \\ +-resume +-profile pasteur \\ +-with-singularity /path/to/pipeline-dir/from/step/3/singularity-images/singularity.img \\ +--email my-email@pasteur.fr \\ +--custom_config_base /path/to/configs/from/step/4/ \\ +-c my-specific.config +... +``` From 64e78a3dc15ef11224b8da0bf7ed64b9c30e816b Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 15:17:25 +0200 Subject: [PATCH 37/79] Update the doc --- docs/pasteur.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pasteur.md b/docs/pasteur.md index 153f4d2..676199d 100644 --- a/docs/pasteur.md +++ b/docs/pasteur.md @@ -43,7 +43,7 @@ module load singularity # Run nextflow workflow nextflow run \\ -/path/to/pipeline-dir/from/step/2/workflow \\ +/path/to/pipeline-dir/from/step/3/workflow \\ -resume -profile pasteur \\ -with-singularity /path/to/pipeline-dir/from/step/3/singularity-images/singularity.img \\ From f9a2baf6143fec9325cd8fea10eb4dcdd478470e Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 16:10:33 +0200 Subject: [PATCH 38/79] Add link to Pasteur doc --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 73b01be..edab0b5 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ Currently documentation is available for the following clusters: * [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UPPMAX](docs/uppmax.md) * [UZH](docs/uzh.md) +* [PASTEUR](docs/pasteur.md) ### Uploading to `nf-core/configs` From be28a34839f795ec9cff95102bc1b26d1ca231ae Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 16:12:30 +0200 Subject: [PATCH 39/79] Updat Pasteur doc --- docs/pasteur.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/pasteur.md b/docs/pasteur.md index 676199d..554761b 100644 --- a/docs/pasteur.md +++ b/docs/pasteur.md @@ -8,14 +8,16 @@ To use, run the pipeline with `-profile pasteur`. This will download and launch ## Running the workflow on the Pasteur cluster ->Nextflow is not installed by default on the Pasteur cluster. +Nextflow is not installed by default on the Pasteur cluster. - Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) ->Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. +Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. Nextflow shouldn't run directly on the submission node but on a compute node. -The compute nodes don't have access to internet so you need to run it offline. To do that: -1. Create a virtualenv to install nf-core in order to get the workflow +The compute nodes don't have access to internet so you need to run it offline. + +To do that: +1. Create a virtualenv to install nf-core ```bash module purge module load Python/3.6.0 From 6298f78bf40d4f2b66f7b316e30ae07a9c7c0b20 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 21:33:47 +0100 Subject: [PATCH 40/79] .travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 918a846..c8db01c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,8 @@ install: - mkdir -p ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests env: - - NXF_VER='18.10.1' # Specify a minimum NF version that should be tested and work - - NXF_VER='' # Plus: get the latest NF version and check, that it works + - NXF_VER='18.10.1' SCRATCH='~' # Specify a minimum NF version that should be tested and work. Set SCRATCH for prince.config. + - NXF_VER='' SCRATCH='~' # Plus: get the latest NF version and check, that it works. Set SCRATCH for prince.config. script: # Run the pipeline with the test profile and test remote config From 840a97ed68b589871ee7e688040ab1b960f4e7bd Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 21:52:16 +0100 Subject: [PATCH 41/79] Clean up and alphabetasize --- README.md | 10 ++++++---- nfcore_custom.config | 8 ++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 675a95e..657b226 100644 --- a/README.md +++ b/README.md @@ -82,24 +82,26 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs Currently documentation is available for the following clusters: -* [BINAC](docs/binac.md) * [BIGPURPLE](docs/bigpurple.md) +* [BINAC](docs/binac.md) * [CCGA](docs/ccga.md) * [CCGA_DX](/docs/ccga_dx.md) * [CFC](docs/binac.md) * [CRICK](docs/crick.md) +* [CZBIOHUB_AWS](docs/czbiohub.md) +* [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) * [MENDEL](docs/mendel.md) * [MUNIN](docs/munin.md) +* [PASTEUR](docs/pasteur.md) * [PHOENIX](docs/phoenix.md) * [PRINCE](docs/prince.md) * [SHH](docs/shh.md) * [UCT_HEX](docs/uct_hex.md) -* [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UPPMAX](docs/uppmax.md) +* [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UZH](docs/uzh.md) -* [PASTEUR](docs/pasteur.md) ### Uploading to `nf-core/configs` @@ -111,4 +113,4 @@ We will be notified automatically when you have created your pull request, and p ## Help -If you have any questions or issues please send us a message on [Slack](https://nf-core-invite.herokuapp.com/). +If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack). diff --git a/nfcore_custom.config b/nfcore_custom.config index 5dd8ee2..2800c1d 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -12,6 +12,7 @@ params.custom_config_version = 'master' params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" profiles { + bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } @@ -21,20 +22,19 @@ profiles { czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" - } + } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } + pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } + prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax-devel.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } - prince { includeConfig "${params.custom_config_base}/conf/prince.config" } - bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } - pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } } // If user hostnames contain one of these substring and they are From daf1b9dcd259659d42753624b537310207880a75 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:09:30 +0100 Subject: [PATCH 42/79] Fix type in uppmax-devel --- README.md | 2 +- conf/{uppmax-devel.config => uppmax_devel.config} | 0 nfcore_custom.config | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename conf/{uppmax-devel.config => uppmax_devel.config} (100%) diff --git a/README.md b/README.md index 657b226..059aa30 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Currently documentation is available for the following clusters: * [SHH](docs/shh.md) * [UCT_HEX](docs/uct_hex.md) * [UPPMAX](docs/uppmax.md) -* [UPPMAX-DEVEL](docs/uppmax-devel.md) +* [UPPMAX_DEVEL](docs/uppmax.md) * [UZH](docs/uzh.md) ### Uploading to `nf-core/configs` diff --git a/conf/uppmax-devel.config b/conf/uppmax_devel.config similarity index 100% rename from conf/uppmax-devel.config rename to conf/uppmax_devel.config diff --git a/nfcore_custom.config b/nfcore_custom.config index 2800c1d..2ae18df 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -32,7 +32,7 @@ profiles { prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } - uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax-devel.config" } + uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From 8be28979617ff6a313c923488e5d7fd2dcb0ca68 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:27:37 +0100 Subject: [PATCH 43/79] Remove problematic configs --- nfcore_custom.config | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 2ae18df..37cd5f1 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -18,11 +18,6 @@ profiles { ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } - czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } - czbiohub_aws_highpriority { - includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" - includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" - } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From 70919d0dd30661bde3c64272785ad2f10afd5bf2 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:28:03 +0100 Subject: [PATCH 44/79] Remove more configs --- nfcore_custom.config | 2 -- 1 file changed, 2 deletions(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 37cd5f1..c19ed92 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -19,12 +19,10 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } - hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } - prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } From a496bfaeef0eade7f157d8be7f1aa260d4879eab Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:33:49 +0100 Subject: [PATCH 45/79] Remove uppmax_devel --- nfcore_custom.config | 1 - 1 file changed, 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index c19ed92..0578add 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -25,7 +25,6 @@ profiles { phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } - uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From 0f6802a7c89705ab61f39dc4170a5f143ff16602 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:37:11 +0100 Subject: [PATCH 46/79] Add hebbe back in --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 0578add..4b79495 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -19,6 +19,7 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } + hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } From f3e61fdd66e1b94c37a9dd42864827d92914cd6e Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:40:43 +0100 Subject: [PATCH 47/79] Add prince back in --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 4b79495..943429a 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -24,6 +24,7 @@ profiles { munin { includeConfig "${params.custom_config_base}/conf/munin.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } + prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } From eb0e89eeaac958b2928558aee106e9a5a7a0a779 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:44:54 +0100 Subject: [PATCH 48/79] Remove SCRATCH variable --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c8db01c..79de276 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,8 @@ install: - mkdir -p ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests env: - - NXF_VER='18.10.1' SCRATCH='~' # Specify a minimum NF version that should be tested and work. Set SCRATCH for prince.config. - - NXF_VER='' SCRATCH='~' # Plus: get the latest NF version and check, that it works. Set SCRATCH for prince.config. + - NXF_VER='18.10.1' # Specify a minimum NF version that should be tested and work. Set SCRATCH for prince.config. + - NXF_VER='' # Plus: get the latest NF version and check, that it works. Set SCRATCH for prince.config. script: # Run the pipeline with the test profile and test remote config From 0326b5603897b0713cff5dee397c837f188b2b46 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:48:22 +0100 Subject: [PATCH 49/79] Add czbiohub back in --- .travis.yml | 4 ++-- nfcore_custom.config | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 79de276..c8db01c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,8 @@ install: - mkdir -p ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests env: - - NXF_VER='18.10.1' # Specify a minimum NF version that should be tested and work. Set SCRATCH for prince.config. - - NXF_VER='' # Plus: get the latest NF version and check, that it works. Set SCRATCH for prince.config. + - NXF_VER='18.10.1' SCRATCH='~' # Specify a minimum NF version that should be tested and work. Set SCRATCH for prince.config. + - NXF_VER='' SCRATCH='~' # Plus: get the latest NF version and check, that it works. Set SCRATCH for prince.config. script: # Run the pipeline with the test profile and test remote config diff --git a/nfcore_custom.config b/nfcore_custom.config index 943429a..6d21f5d 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -18,6 +18,7 @@ profiles { ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } + czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From 7ee70fb742696080266dc6cadde7e91ba66252ca Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:52:20 +0100 Subject: [PATCH 50/79] Add in uppmax-devel --- nfcore_custom.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 6d21f5d..eb0abf4 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -18,7 +18,7 @@ profiles { ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } - czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } + gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } @@ -29,6 +29,7 @@ profiles { shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } + uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From a692ded933e3871a07eda1ceee2db51138801905 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 23:30:51 +0100 Subject: [PATCH 51/79] Remove workDir in czbiohub --- conf/czbiohub_aws.config | 2 +- nfcore_custom.config | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index 1dd8da2..576e49e 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -24,7 +24,7 @@ process { errorStrategy = 'ignore' } -workDir = "s3://czb-nextflow/intermediates/" +//workDir = "s3://czb-nextflow/intermediates/" aws.region = 'us-west-2' executor.awscli = '/home/ec2-user/miniconda/bin/aws' diff --git a/nfcore_custom.config b/nfcore_custom.config index eb0abf4..4180c1d 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -18,7 +18,8 @@ profiles { ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } - + czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } + czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } @@ -29,7 +30,6 @@ profiles { shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } - uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From e0c684ee8f1bc7be315659ba582737b15b71ee53 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 23:39:42 +0100 Subject: [PATCH 52/79] Aarrrgh pulling configs from master --- conf/czbiohub_aws.config | 2 +- nfcore_custom.config | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index 576e49e..1dd8da2 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -24,7 +24,7 @@ process { errorStrategy = 'ignore' } -//workDir = "s3://czb-nextflow/intermediates/" +workDir = "s3://czb-nextflow/intermediates/" aws.region = 'us-west-2' executor.awscli = '/home/ec2-user/miniconda/bin/aws' diff --git a/nfcore_custom.config b/nfcore_custom.config index 4180c1d..2d4c183 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -30,6 +30,7 @@ profiles { shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } + uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From 91a5e64e1cd0ab7944a3340617f457fe7da5aecc Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 1 Aug 2019 09:42:50 +0100 Subject: [PATCH 53/79] Ignore czbiohub_aws --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c8db01c..4985029 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,4 +26,5 @@ script: grep "{.*includeConfig.*[a-z]*\.config\"" ${TRAVIS_BUILD_DIR}/nfcore_custom.config | \ tr -s ' ' | \ cut -d " " -f 2 | \ + grep -v "czbiohub_aws" | \ xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf -profile {} From c583aa0233bf5fd5769db4db3046fd66a5fb99e8 Mon Sep 17 00:00:00 2001 From: Martin Proks Date: Mon, 5 Aug 2019 11:19:06 +0200 Subject: [PATCH 54/79] Added kraken config (#15) * Added kraken template * docs: updated some info * fix: updated igenomes_base path --- conf/kraken.config | 24 ++++++++++++++++++++++++ docs/kraken.md | 10 ++++++++++ 2 files changed, 34 insertions(+) create mode 100644 conf/kraken.config create mode 100644 docs/kraken.md diff --git a/conf/kraken.config b/conf/kraken.config new file mode 100644 index 0000000..59417e9 --- /dev/null +++ b/conf/kraken.config @@ -0,0 +1,24 @@ +//Profile config names for nf-core/configs +params { + config_profile_name = 'KRAKEN' + config_profile_description = 'Jenkins cluster provided by nf-core/configs.' + config_profile_contact = 'Maxime Garcia or Johannes Alneberg' + config_profile_url = 'kraken.dyn.scilifelab.se' +} + +process { + executor = 'local' +} + +docker { + enabled = true + mountFlags = 'z' + fixOwnership = true +} + +params { + max_memory = 60.GB + max_cpus = 16 + max_time = 72.h + igenomes_base = '/share/igenomes/' +} diff --git a/docs/kraken.md b/docs/kraken.md new file mode 100644 index 0000000..9426087 --- /dev/null +++ b/docs/kraken.md @@ -0,0 +1,10 @@ +# nf-core/configs: KRAKEN Configuration + +This profile can be **only** combined with `jenkins.config`. It is used for +testing pipeline with real data on **in-house** cluster located at SciLifeLab. + +To use, run the pipeline with `-profile kraken`. This will download and launch +the [`kraken.config`](../conf/kraken.config) which has been pre-configured to +test the pipeline using `docker` by default. + +Example: `nextflow run -profile kraken,jenkins` From da2a9047b000947805d2540aadbaba601a02c274 Mon Sep 17 00:00:00 2001 From: Martin Proks Date: Mon, 5 Aug 2019 14:59:36 +0200 Subject: [PATCH 55/79] fix: added kraken to the list (#64) --- README.md | 1 + nfcore_custom.config | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 059aa30..6e68fb4 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ Currently documentation is available for the following clusters: * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) +* [KRAKEN](docs/kraken.md) * [MENDEL](docs/mendel.md) * [MUNIN](docs/munin.md) * [PASTEUR](docs/pasteur.md) diff --git a/nfcore_custom.config b/nfcore_custom.config index 2d4c183..9e7d190 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -22,6 +22,7 @@ profiles { czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } + kraken { includeConfig "${params.custom_config_base}/conf/kraken.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } From 332ba0dadb93b0813ac36c9b937c4d8b24fc0b48 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 7 Aug 2019 11:10:48 +0200 Subject: [PATCH 56/79] fix: update and rearrange munin configuration file and docs --- conf/munin.config | 20 +++++++++++--------- docs/munin.md | 6 ++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/conf/munin.config b/conf/munin.config index bca56f7..58e10f7 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -1,20 +1,22 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'Big iron cluster profile provided by nf-core/configs.' + config_profile_description = 'Munin profile provided by nf-core/configs.' config_profile_contact = 'Szilveszter Juhos (@szilva)' config_profile_url = '' } process { executor = 'local' + maxForks = 48 } +// To use docker, use nextflow run -profile munin,singularity singularity { enabled = true autoMounts = true } -// To use docker instead of singularity, use nextflow run -profile munin,docker +// To use docker, use nextflow run -profile munin,docker docker { enabled = false mountFlags = 'z' @@ -22,11 +24,11 @@ docker { } params { - saveReference = true - - max_memory = 128.GB - max_cpus = 16 + // general params + max_memory = 754.GB + max_cpus = 48 max_time = 72.h - // illumina iGenomes reference file paths on UPPMAX - igenomes_base = '/data1/references/igenomes/' -} + + // AWS iGenomes reference file paths on munin + igenome_base = '/data1/references/igenomes/' +} \ No newline at end of file diff --git a/docs/munin.md b/docs/munin.md index 17233bb..7f164ce 100644 --- a/docs/munin.md +++ b/docs/munin.md @@ -1,8 +1,10 @@ # nf-core/configs: MUNIN Configuration -All nf-core pipelines have been successfully configured for use on the MUNIN cluster aka big iron. +All nf-core pipelines have been successfully configured for use on the MUNIN cluster. -To use, run the pipeline with `-profile munin`. This will download and launch the [`munin.config`](../conf/munin.config) which has been pre-configured with a setup suitable for the MUNIN cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile munin`. +This will download and launch the [`munin.config`](../conf/munin.config) which has been pre-configured with a setup suitable for the MUNIN cluster. +Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Example: `nextflow run -profile munin` From 12fb2a60c57e85b066d10c18e006759773fe773d Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 7 Aug 2019 11:47:41 +0200 Subject: [PATCH 57/79] Update conf/munin.config Co-Authored-By: Harshil Patel --- conf/munin.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/munin.config b/conf/munin.config index 58e10f7..d837b08 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -10,7 +10,7 @@ process { maxForks = 48 } -// To use docker, use nextflow run -profile munin,singularity +// To use singularity, use nextflow run -profile munin,singularity singularity { enabled = true autoMounts = true @@ -31,4 +31,4 @@ params { // AWS iGenomes reference file paths on munin igenome_base = '/data1/references/igenomes/' -} \ No newline at end of file +} From d6a763214cc8dfa09893a1ea1079b6bad8216f9b Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 7 Aug 2019 11:47:47 +0200 Subject: [PATCH 58/79] Update conf/munin.config Co-Authored-By: Harshil Patel --- conf/munin.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/munin.config b/conf/munin.config index d837b08..24b5dac 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -29,6 +29,6 @@ params { max_cpus = 48 max_time = 72.h - // AWS iGenomes reference file paths on munin + // Local AWS iGenomes reference file paths on munin igenome_base = '/data1/references/igenomes/' } From 0bd19517b30a1a79b27db1d1163293d38f04ef41 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 7 Aug 2019 11:55:56 +0200 Subject: [PATCH 59/79] fix: typos --- conf/munin.config | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/conf/munin.config b/conf/munin.config index 58e10f7..b1e4628 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -1,6 +1,6 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'Munin profile provided by nf-core/configs.' + config_profile_description = 'MUNIN profile provided by nf-core/configs.' config_profile_contact = 'Szilveszter Juhos (@szilva)' config_profile_url = '' } @@ -10,10 +10,9 @@ process { maxForks = 48 } -// To use docker, use nextflow run -profile munin,singularity +// To use singularity, use nextflow run -profile munin,singularity singularity { enabled = true - autoMounts = true } // To use docker, use nextflow run -profile munin,docker @@ -29,6 +28,6 @@ params { max_cpus = 48 max_time = 72.h - // AWS iGenomes reference file paths on munin + // Local AWS iGenomes reference file paths igenome_base = '/data1/references/igenomes/' } \ No newline at end of file From f69524d59a3afdfcb86ddb405f954f27833f3608 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 7 Aug 2019 11:56:10 +0200 Subject: [PATCH 60/79] feat: update docs --- docs/munin.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/munin.md b/docs/munin.md index 7f164ce..167ff9b 100644 --- a/docs/munin.md +++ b/docs/munin.md @@ -2,16 +2,30 @@ All nf-core pipelines have been successfully configured for use on the MUNIN cluster. +## Usage + To use, run the pipeline with `-profile munin`. + This will download and launch the [`munin.config`](../conf/munin.config) which has been pre-configured with a setup suitable for the MUNIN cluster. -Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Example: `nextflow run -profile munin` -## Docker +### Singularity + +This is the default behavior of this configuration profile. + +Using this profile, if no singularity image are available, one will be downloaded from dockerhub, and converted to a Singularity image before execution of the pipeline. + +It is also possible to specify the singularity profile: + +Example: `nextflow run -profile munin,singularity` + +### Docker It is also possible to execute the pipeline using Docker. +Using this profile, if no docker image are available, one will be downloaded from dockerhub before execution of the pipeline. + Example: `nextflow run -profile munin,docker` ## Below are non-mandatory information on iGenomes specific configuration From 4183090379a031e2c5031ab45aa62f1035f88691 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 7 Aug 2019 12:26:51 +0200 Subject: [PATCH 61/79] fix: reduce cpu and mem usage --- conf/munin.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/munin.config b/conf/munin.config index 9076e38..52062e5 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -7,7 +7,7 @@ params { process { executor = 'local' - maxForks = 48 + maxForks = 46 } // To use singularity, use nextflow run -profile munin,singularity @@ -24,8 +24,8 @@ docker { params { // general params - max_memory = 754.GB - max_cpus = 48 + max_memory = 752.GB + max_cpus = 46 max_time = 72.h // Local AWS iGenomes reference file paths on munin From 486cdfce5f7db8e53372ed92ab53190f0d368a80 Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Wed, 7 Aug 2019 13:19:16 +0200 Subject: [PATCH 62/79] Updating CCGA configs - adding cache dir and mount points --- conf/ccga.config | 1 + conf/ccga_dx.config | 1 + 2 files changed, 2 insertions(+) diff --git a/conf/ccga.config b/conf/ccga.config index b9130e0..6163626 100644 --- a/conf/ccga.config +++ b/conf/ccga.config @@ -14,6 +14,7 @@ params { singularity { enabled = true runOptions = "-B /ifs -B /scratch -B /work_beegfs" + cacheDir = "/ifs/data/nfs_share/ikmb_repository/singularity_cache/" } executor { diff --git a/conf/ccga_dx.config b/conf/ccga_dx.config index 0af95e7..a5c600b 100644 --- a/conf/ccga_dx.config +++ b/conf/ccga_dx.config @@ -13,6 +13,7 @@ params { singularity { enabled = true + runOoptions = "-B /mnt" } executor { From c3c0d088dd01938faa06681af5dd16d747aa08a8 Mon Sep 17 00:00:00 2001 From: Szilveszter Juhos Date: Fri, 16 Aug 2019 11:13:32 +0200 Subject: [PATCH 63/79] typo --- conf/munin.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/munin.config b/conf/munin.config index 52062e5..f53def4 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -29,5 +29,5 @@ params { max_time = 72.h // Local AWS iGenomes reference file paths on munin - igenome_base = '/data1/references/igenomes/' + igenomes_base = '/data1/references/igenomes/' } From 3b638e9ccdb93ab29c95b80fe5c2c1fb76e6b56d Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Tue, 3 Sep 2019 15:37:15 +0200 Subject: [PATCH 64/79] Fixing typo in ccga_dx config --- conf/ccga_dx.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/ccga_dx.config b/conf/ccga_dx.config index a5c600b..1f92b0d 100644 --- a/conf/ccga_dx.config +++ b/conf/ccga_dx.config @@ -13,7 +13,7 @@ params { singularity { enabled = true - runOoptions = "-B /mnt" + runOptions = "-B /mnt" } executor { From 5e257e7e9a38787e3a440a76d695d28c5183f895 Mon Sep 17 00:00:00 2001 From: phue Date: Sun, 22 Sep 2019 16:45:09 +0200 Subject: [PATCH 65/79] add cbe profile --- README.md | 1 + conf/cbe.config | 32 ++++++++++++++++++++++++++++++++ docs/cbe.md | 18 ++++++++++++++++++ nfcore_custom.config | 1 + 4 files changed, 52 insertions(+) create mode 100755 conf/cbe.config create mode 100644 docs/cbe.md diff --git a/README.md b/README.md index 6e68fb4..9a26324 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ Currently documentation is available for the following clusters: * [BIGPURPLE](docs/bigpurple.md) * [BINAC](docs/binac.md) +* [CBE](docs/cbe.md) * [CCGA](docs/ccga.md) * [CCGA_DX](/docs/ccga_dx.md) * [CFC](docs/binac.md) diff --git a/conf/cbe.config b/conf/cbe.config new file mode 100755 index 0000000..5e6aa86 --- /dev/null +++ b/conf/cbe.config @@ -0,0 +1,32 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'CLIP BATCH ENVIRONMENT (CBE) cluster profile provided by nf-core/configs' + config_profile_contact = 'Patrick Hüther (@phue)' + config_profile_url = 'http://www.gmi.oeaw.ac.at/' +} + +process { + executor = 'slurm' + module = 'singularity/3.1.0' + queue = 'c' +} + +singularity.enabled = true + +params { + target_qos = 'medium' + params.max_cpus = 36 + params.max_memory = 170.GB + igenomesIgnore = true +} + +if (params.target_qos == 'short') { + params.max_time = 8.h + process.clusterOptions = '--qos short' +} else if (params.target_qos == 'medium') { + params.max_time = 2.d + process.clusterOptions = '--qos medium' +} else { + params.max_time = 14.d + process.clusterOptions = '--qos long' +} diff --git a/docs/cbe.md b/docs/cbe.md new file mode 100644 index 0000000..5cfa5b9 --- /dev/null +++ b/docs/cbe.md @@ -0,0 +1,18 @@ +# nf-core/configs: CBE Configuration + +All nf-core pipelines have been successfully configured for use on the CLIP BATCH ENVIRONMENT (CBE) cluster at the Vienna BioCenter (VBC). + +To use, run the pipeline with `-profile cbe`. This will download and launch the [`cbe.config`](../conf/cbe.config) which has been pre-configured with a setup suitable for the CBE cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on CBE. You can do this by issuing the commands below: + +```bash +## Load Nextflow and Singularity environment modules +module purge +module load nextflow/19.04.0 +module load singularity/3.1.0 +``` + +>NB: You will need an account to use the HPC cluster on CBE in order to run the pipeline. If in doubt contact IT. + +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/nfcore_custom.config b/nfcore_custom.config index 9e7d190..6e320c4 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -14,6 +14,7 @@ params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/$ profiles { bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } + cbe { includeConfig "${params.custom_config_base}/conf/cbe.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } From 014111b925f3d3ebc40eb034e71ee6af06b35813 Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 23 Sep 2019 13:42:11 +0200 Subject: [PATCH 66/79] point travis to local clone Previously travis tests could only succeed if the config was already present in the master branch, but for newly added configs it would always fail. This commit sets custom_config_base to the local clone of the PR branch on the Travis VM where the file is present. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4985029..60bfc96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,4 +27,4 @@ script: tr -s ' ' | \ cut -d " " -f 2 | \ grep -v "czbiohub_aws" | \ - xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf -profile {} + xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf --custom_config_base=${TRAVIS_BUILD_DIR} -profile {} From d7e891a5765074259a060c8cdce2ff7b3058bcc7 Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 23 Sep 2019 13:55:43 +0200 Subject: [PATCH 67/79] Revert "point travis to local clone" This reverts commit 014111b925f3d3ebc40eb034e71ee6af06b35813. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 60bfc96..4985029 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,4 +27,4 @@ script: tr -s ' ' | \ cut -d " " -f 2 | \ grep -v "czbiohub_aws" | \ - xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf --custom_config_base=${TRAVIS_BUILD_DIR} -profile {} + xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf -profile {} From fb60856e409883d7d1f9cdae808827be1e4d5926 Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 23 Sep 2019 13:57:58 +0200 Subject: [PATCH 68/79] point travis to local clone Previously travis tests could only succeed if the config was already present in the master branch, but for newly added configs it would always fail. This commit sets custom_config_base to the local clone of the PR branch on the Travis VM where the file is present. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4985029..9da38dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,4 +27,4 @@ script: tr -s ' ' | \ cut -d " " -f 2 | \ grep -v "czbiohub_aws" | \ - xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf -profile {} + xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf --custom_config_base=${TRAVIS_BUILD_DIR} -profile {} From abb67800816687319bc6febb9d2ffe08412c04e8 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 12:48:24 +0200 Subject: [PATCH 69/79] Add Singularity Cache Directory --- conf/cfc.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/cfc.config b/conf/cfc.config index 6fc3b26..0928c00 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -7,6 +7,7 @@ params { singularity { enabled = true + cacheDir = '/nfsmounts/containers' } process { From 4555dd7d36b7c0b83f0d2bd63166f2cf1fca89af Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 12:51:01 +0200 Subject: [PATCH 70/79] Update conf/cfc.config --- conf/cfc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index 0928c00..77a1599 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -7,7 +7,7 @@ params { singularity { enabled = true - cacheDir = '/nfsmounts/containers' + cacheDir = '/nfsmounts/container' } process { From 5fae33669d469fd266b1e87b8aaad5dcffcede43 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 13:07:28 +0200 Subject: [PATCH 71/79] Update cfc.config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Try this out 👍 --- conf/cfc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index 77a1599..d1f1112 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -7,7 +7,7 @@ params { singularity { enabled = true - cacheDir = '/nfsmounts/container' + cacheDir = "/nfsmounts/container/nf-core/${workflow.manifest.name}/v${workflow.manifest.version}/" } process { From 865909ac54d76078fc2c58aed07221acbc14a674 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 13:14:21 +0200 Subject: [PATCH 72/79] Update cfc.config --- conf/cfc.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index d1f1112..75129b6 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -7,7 +7,7 @@ params { singularity { enabled = true - cacheDir = "/nfsmounts/container/nf-core/${workflow.manifest.name}/v${workflow.manifest.version}/" + cacheDir = params.singularity_cachedir } process { @@ -22,6 +22,7 @@ weblog{ params { igenomes_base = '/nfsmounts/igenomes' + singularity_cachedir = "/nfsmounts/container/nf-core/${workflow.manifest.name}/v${workflow.manifest.version}/" max_memory = 60.GB max_cpus = 20 max_time = 140.h From 1e54d9510dfa6c1be09fb0c21a59c6fece29d846 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 13:15:49 +0200 Subject: [PATCH 73/79] Update cfc.config --- conf/cfc.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/cfc.config b/conf/cfc.config index 75129b6..c852fdd 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -5,9 +5,10 @@ params { config_profile_url = 'http://qbic.uni-tuebingen.de/' } +CACHE_DIR = "/nfsmounts/container/nf-core/${workflow.manifest.name}/v${workflow.manifest.version}/" singularity { enabled = true - cacheDir = params.singularity_cachedir + cacheDir = CACHE_DIR } process { @@ -22,7 +23,6 @@ weblog{ params { igenomes_base = '/nfsmounts/igenomes' - singularity_cachedir = "/nfsmounts/container/nf-core/${workflow.manifest.name}/v${workflow.manifest.version}/" max_memory = 60.GB max_cpus = 20 max_time = 140.h From ed8c9d9e7a066e0ab10cced039b8f7949f9ddb7a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 13:20:58 +0200 Subject: [PATCH 74/79] Update cfc.config --- conf/cfc.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/cfc.config b/conf/cfc.config index c852fdd..77a1599 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -5,10 +5,9 @@ params { config_profile_url = 'http://qbic.uni-tuebingen.de/' } -CACHE_DIR = "/nfsmounts/container/nf-core/${workflow.manifest.name}/v${workflow.manifest.version}/" singularity { enabled = true - cacheDir = CACHE_DIR + cacheDir = '/nfsmounts/container' } process { From 855da0cee35054c80bdf43bb012e561a22fac8a1 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 8 Oct 2019 14:19:42 +0200 Subject: [PATCH 75/79] Refactors weblog url --- conf/binac.config | 5 +++++ conf/cfc.config | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index 0e69d4a..6049f78 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -21,3 +21,8 @@ params { max_cpus = 28 max_time = 48.h } + +weblog{ + enabled = true + url = 'http://services.qbic.uni-tuebingen.de:8080/flowstore/workflows' +} diff --git a/conf/cfc.config b/conf/cfc.config index 77a1599..d675648 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -17,7 +17,7 @@ process { weblog{ enabled = true - url = 'http://services.qbic.uni-tuebingen.de:8080/workflowservice/workflows' + url = 'http://services.qbic.uni-tuebingen.de:8080/flowstore/workflows' } params { From dc3faf8a5fe9b63d0c04bbc540a1ce46843f6bbb Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 10 Oct 2019 12:56:39 +0200 Subject: [PATCH 76/79] Sends weblog messages over HTTPS --- conf/binac.config | 2 +- conf/cfc.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/binac.config b/conf/binac.config index 6049f78..9447ca4 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -24,5 +24,5 @@ params { weblog{ enabled = true - url = 'http://services.qbic.uni-tuebingen.de:8080/flowstore/workflows' + url = 'https://services.qbic.uni-tuebingen.de/flowstore/workflows' } diff --git a/conf/cfc.config b/conf/cfc.config index d675648..d5b1c3b 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -17,7 +17,7 @@ process { weblog{ enabled = true - url = 'http://services.qbic.uni-tuebingen.de:8080/flowstore/workflows' + url = 'https://services.qbic.uni-tuebingen.de/flowstore/workflows' } params { From 9b3fda36f65bfb1a2f60793df4f7b38f97bb1231 Mon Sep 17 00:00:00 2001 From: phue Date: Thu, 10 Oct 2019 21:22:19 +0200 Subject: [PATCH 77/79] update singularity version singularity/3.1.0 is not available anymore --- conf/cbe.config | 2 +- docs/cbe.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index 5e6aa86..c303f92 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -7,7 +7,7 @@ params { process { executor = 'slurm' - module = 'singularity/3.1.0' + module = 'singularity/3.2.1' queue = 'c' } diff --git a/docs/cbe.md b/docs/cbe.md index 5cfa5b9..8a84294 100644 --- a/docs/cbe.md +++ b/docs/cbe.md @@ -10,7 +10,7 @@ Before running the pipeline you will need to load Nextflow and Singularity using ## Load Nextflow and Singularity environment modules module purge module load nextflow/19.04.0 -module load singularity/3.1.0 +module load singularity/3.2.1 ``` >NB: You will need an account to use the HPC cluster on CBE in order to run the pipeline. If in doubt contact IT. From 1f2786c78f24742f0cb719595efc0e275bc0a1c9 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 16 Oct 2019 18:15:04 +0200 Subject: [PATCH 78/79] add genouest cluster --- README.md | 1 + conf/genouest.config | 23 +++++++++++++++++++++++ docs/genouest.md | 36 ++++++++++++++++++++++++++++++++++++ nfcore_custom.config | 2 ++ 4 files changed, 62 insertions(+) create mode 100644 conf/genouest.config create mode 100644 docs/genouest.md diff --git a/README.md b/README.md index 9a26324..451c12f 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ Currently documentation is available for the following clusters: * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) +* [GENOUEST](docs/genouest.md) * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) * [KRAKEN](docs/kraken.md) diff --git a/conf/genouest.config b/conf/genouest.config new file mode 100644 index 0000000..68d82a6 --- /dev/null +++ b/conf/genouest.config @@ -0,0 +1,23 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'The GenOuest cluster profile' + config_profile_contact = 'Anthony Bretaudeau (@abretaud)' + config_profile_url = 'https://www.genouest.org' +} + +singularity { + enabled = true + autoMounts = true + runOptions = '-B /scratch:/scratch -B /local:/local -B /db:/db' +} + +process { + executor = 'slurm' +} + +params { + igenomesIgnore = true + max_memory = 750.GB + max_cpus = 80 + max_time = 336.h +} diff --git a/docs/genouest.md b/docs/genouest.md new file mode 100644 index 0000000..9c37553 --- /dev/null +++ b/docs/genouest.md @@ -0,0 +1,36 @@ +# nf-core/configs: GenOuest Configuration + +All nf-core pipelines have been successfully configured for use on the GenOuest cluster. + +To use, run the pipeline with `-profile genouest`. This will download and launch the [`genouest.config`](../conf/genouest.config) which has been pre-configured with a setup suitable for the GenOuest cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +## Running the workflow on the GenOuest cluster + +Nextflow is installed on the GenOuest cluster. You need to activate it like this: + +```bash +source /local/env/envnextflow-19.07.0.sh +``` + +Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. +Nextflow shouldn't run directly on the submission node but on a compute node. Run nextflow from a compute node: + +```bash +# Login to a compute node +srun --pty bash + +# Load the dependencies if not done before +source /local/env/envnextflow-19.07.0.sh + +# Run a downloaded/git-cloned nextflow workflow from +nextflow run \\ +/path/to/nf-core/workflow \\ +-resume +-profile genouest \\ +--email my-email@example.org \\ +-c my-specific.config +... + +# Or use the nf-core client +nextflow run nf-core/rnaseq ... +``` diff --git a/nfcore_custom.config b/nfcore_custom.config index 6e320c4..06182f5 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -21,6 +21,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } + genouest { includeConfig "${params.custom_config_base}/conf/genouest.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } kraken { includeConfig "${params.custom_config_base}/conf/kraken.config" } @@ -43,6 +44,7 @@ params { // This is a groovy map, not a nextflow parameter set hostnames = [ crick: ['.thecrick.org'], + genouest: ['.genouest.org'], uppmax: ['.uppmax.uu.se'] ] } From 883a761651739b9704b76b386523ee5eb0d072ed Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 16 Oct 2019 18:45:26 +0200 Subject: [PATCH 79/79] add link --- docs/genouest.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/genouest.md b/docs/genouest.md index 9c37553..85341ef 100644 --- a/docs/genouest.md +++ b/docs/genouest.md @@ -6,7 +6,9 @@ To use, run the pipeline with `-profile genouest`. This will download and launch ## Running the workflow on the GenOuest cluster -Nextflow is installed on the GenOuest cluster. You need to activate it like this: +Nextflow is installed on the GenOuest cluster. Some documentation is available on the [GenOuest website](https://www.genouest.org/howto/#nextflow). + +You need to activate it like this: ```bash source /local/env/envnextflow-19.07.0.sh