From 13df59bfaf45d385fc7d745b31b9a174731da0ef Mon Sep 17 00:00:00 2001 From: phue Date: Tue, 12 Mar 2019 17:57:31 +0000 Subject: [PATCH 01/40] remove singularity support reasons: * the kernel running on the compute nodes does not support overlayfs und thus containers downloaded from public registries won't run because they lack the /lustre mountpoint * building containers manually using the Singularity file would work, however nf-core pipelines will stop providing these files in future releases --- conf/mendel.config | 2 +- docs/mendel.md | 18 +++--------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/conf/mendel.config b/conf/mendel.config index 940a914..0715b42 100644 --- a/conf/mendel.config +++ b/conf/mendel.config @@ -11,7 +11,7 @@ manifest { } process { - beforeScript = {'module load Singularity; module load Miniconda3'} + beforeScript = 'module load Miniconda3/4.6.7' executor = 'pbspro' clusterOptions = { "-P $params.project" } } diff --git a/docs/mendel.md b/docs/mendel.md index 3f3db20..ae0ad4c 100644 --- a/docs/mendel.md +++ b/docs/mendel.md @@ -2,27 +2,15 @@ All nf-core pipelines have been successfully configured for use on the MENDEL CLUSTER at the Gregor Mendel Institute (GMI). -To use, run the pipeline with `-profile conda,mendel`. This will download and launch the [`mendel.config`](../conf/mendel.config) which has been pre-configured with a setup suitable for the MENDEL cluster. A Conda environment will be created automatically and software dependencies will be downloaded from ['bioconda'](https://bioconda.github.io/). +To use, run the pipeline with `-profile conda,mendel`. This will download and launch the [`mendel.config`](../conf/mendel.config) which has been pre-configured with a setup suitable for the MENDEL cluster. A Conda environment will be created automatically and software dependencies will be resolved via [bioconda](https://bioconda.github.io/). -Theoretically, using `-profile singularity,mendel` would download a docker image containing all of the required software, and convert it to a Singularity image before execution of the pipeline. However, there is a regression in the Singularity deployment on MENDEL which renders containers downloaded from public repositories unusable because they lack the /lustre mountpoint. - -If you want to run the pipeline containerized anyway you will have to build the image yourself (on a machine where you have root access) using the provided `Singularity` file in the pipeline repository: - -```bash -cd /path/to/pipeline-repository -echo 'mkdir /lustre > Singularity' -singularity build nf-core-methylseq-custom.simg Singularity -``` - -After you copied the container image to the cluster filesystem, make sure to pass the path to the image to the pipeline with `-with-singularity /path/to/nf-core-methylseq-custom.simg` - -Before running the pipeline you will need to load Nextflow and Conda using the environment module system on MENDEL. You can do this by issuing the commands below: +Before running the pipeline you will need to load Conda using the environment module system on MENDEL. You can do this by issuing the commands below: ```bash ## Load Nextflow and Conda environment modules module purge module load Nextflow -module load Miniconda3 # not needed if using Singularity +module load Miniconda/4.6.7 ``` >NB: You will need an account to use the HPC cluster in order to run the pipeline. If in doubt contact the HPC team. From aaa7be049a9589e79cce863fd6d582fb8fcf7799 Mon Sep 17 00:00:00 2001 From: KochTobi Date: Sat, 16 Mar 2019 14:11:59 +0100 Subject: [PATCH 02/40] Update cfc config * Change singularity version documentation --- docs/cfc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cfc.md b/docs/cfc.md index e1cccac..5cf87d1 100644 --- a/docs/cfc.md +++ b/docs/cfc.md @@ -10,7 +10,7 @@ Before running the pipeline you will need to load Nextflow and Singularity using ## Load Nextflow and Singularity environment modules module purge module load devel/java_jdk/1.8.0u121 -module load qbic/singularity_slurm/3.0.1 +module load qbic/singularity_slurm/3.0.3 ``` From 441ddbcdab01871842451bf4b33d757d8c63d87a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 28 May 2019 15:25:10 +0200 Subject: [PATCH 03/40] Newest Update is 3.0.3 --- conf/binac.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index 5df94bf..0e69d4a 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -10,7 +10,7 @@ singularity { } process { - beforeScript = 'module load devel/singularity/3.0.1' + beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' } From 66eeeebe81d1aef1d8083e492484e47975d8760b Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Thu, 30 May 2019 11:15:38 -0700 Subject: [PATCH 04/40] Initial commit of CZBiohub --- conf/czbiohub.config | 98 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 conf/czbiohub.config diff --git a/conf/czbiohub.config b/conf/czbiohub.config new file mode 100644 index 0000000..ff1523e --- /dev/null +++ b/conf/czbiohub.config @@ -0,0 +1,98 @@ +/* + * ------------------------------------------------- + * Nextflow config file for UPPMAX (milou / irma) + * ------------------------------------------------- + * Defines reference genomes, using iGenome paths + * Imported under the default 'standard' Nextflow + * profile in nextflow.config + */ + +docker { + enabled = true +} + +process { + executor = 'awsbatch' +} + +params { + saveReference = true + + // Largest SPOT instances available on AWS: https://ec2instances.info/ + max_memory = 1952.GB + max_cpus = 96 + max_time = 240.h + + // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + igenomes_base = "s3://czbiohub-reference/igenomes" + + // GENCODE (human + mouse) reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + gencode_base = "s3://czbiohub-reference/gencode" + transgenes_base = "s3://czbiohub-reference/transgenes" + + // AWS configurations + awsregion = "us-west-2" + awsqueue = "nextflow" + workDir = "s3://czb-nextflow/rnaseq/" + + // GENCODE GTF and fasta files + genomes { + 'GRCh38' { + fasta = "${params.gencode_base}/human/v29/GRCh38.p12.genome.fa.gz" + gtf = "${params.gencode_base}/human/v29/gencode.vM19.annotation.gtf.gz" + } + 'GRCm38' { + fasta = "${params.gencode_base}/mouse/vM19/GRCm38.p6.genome.fa.gz" + gtf = "${params.gencode_base}/mouse/vM19/gencode.vM19.annotation.gtf.gz" + } + } + + transgenes { + 'ChR2' { + fasta = "${params.transgenes_base}/ChR2/ChR2.fa.gz" + gtf = "${params.transgenes_base}/ChR2/ChR2.gtf.gz" + } + 'Cre' { + fasta = "${params.transgenes_base}/Cre/Cre.fa.gz" + gtf = "${params.transgenes_base}/Cre/Cre.gtf.gz" + } + 'ERCC' { + fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa.gz" + gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf.gz" + } + 'GCaMP6m' { + fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa.gz" + gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf.gz" + } + 'GFP' { + fasta = "${params.transgenes_base}/Gfp/Gfp.fa.gz" + gtf = "${params.transgenes_base}/Gfp/Gfp.gtf.gz" + } + 'NpHR' { + fasta = "${params.transgenes_base}/NpHR/NpHR.fa.gz" + gtf = "${params.transgenes_base}/NpHR/NpHR.gtf.gz" + } + 'RCaMP' { + fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa.gz" + gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf.gz" + } + 'RGECO' { + fasta = "${params.transgenes_base}/RGECO/RGECO.fa.gz" + gtf = "${params.transgenes_base}/RGECO/RGECO.gtf.gz" + } + 'Tdtom' { + fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa.gz" + gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf.gz" + } + 'Car-T' { + fasta = "${params.transgenes_base}/car-t/car-t.fa.gz" + gtf = "${params.transgenes_base}/car-t/car-t.gtf.gz" + } + 'zsGreen' { + fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa.gz" + gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf.gz" + } + } +} From ac804859f42a2bfa20a7f9c6fbbc27915729ed47 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 31 May 2019 00:47:06 +0100 Subject: [PATCH 05/40] Add nf-core logo --- README.md | 4 +--- docs/images/nfcore-configs_logo.png | Bin 0 -> 14072 bytes 2 files changed, 1 insertion(+), 3 deletions(-) create mode 100755 docs/images/nfcore-configs_logo.png diff --git a/README.md b/README.md index 6a53642..75d6199 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ - - -# [nf-core/configs](https://github.com/nf-core/configs) +# ![nf-core/configs](docs/images/nfcore-configs_logo.png) [![Build Status](https://travis-ci.org/nf-core/configs.svg?branch=master)](https://travis-ci.org/nf-core/configs) diff --git a/docs/images/nfcore-configs_logo.png b/docs/images/nfcore-configs_logo.png new file mode 100755 index 0000000000000000000000000000000000000000..eef073868983145737377f81cb92f40132f3c866 GIT binary patch literal 14072 zcmaKTWmp_d(>3ny?gY2s9(?ft2`(Gl-QC@mU_leIgy8PBIKeHrySwYVxu5U%x7Vys zb$9htcTJyjI_9IA0wx+68Vn2!rXoOA69xu$3i^2v1qu3Us1FQ+z9G0tD}F+OKKxP4 zqoLnXodJ4oFfd3}|DLdJf}0!AA4%QibltU_EZx1nxLUw?d3kZ#ING_HeQ~znbaJ)I zJ{2Q_fuVs>l$HAAopX}wv^Ebr+^Rq}@wV^zLb^ES_C?aY3|Jw06oT z--@bqr2Vd}UUVFlVkb-H4%8+Vk6r;$v?0k`6_HV$CFoG zgFsFrWq6~;-vI1U2nC~pg2?_QPKq#k?+Rzt`OFtxHzv#`B{)Rd^N*%Wc{lU4xY^sMclgQ7~(ocU~sUpEhzIz_|mh^Ri zwT||GjZd-MuKGgB{>lzyg4yCdT=>RI%{hDTD_;Wb%T$4WW9iu#IIY+NS*f3%P+yfm z2Ep)f%U<-(P9Y94X&`!qUnez;Yh!_S0b3?o_T)1lR;U?KD-1DwF~yw&EL@RuD*JFc zW#3-uB(aJ2T^MPffK=muCB-=0U39A55k_zpl7<+%S zjXT+V2-LAt(ZSXvNw;8Ep87UZMBUN46%|i?Mu7N>Id%%+GXxR#ti3Qfk^uoe=i^%f z|0}Nx=??_pJh;+pUbDd>m50LV3xj9&^Uqyf_*=rQZ+2Gs?=K!g$%~nl?DjO*lVDH% zVDC+#&gAw)VlPbBt}ZP&=}6qIj^qjbstT9V0SnK2JT?Uy8|}t&*b8G@J2@DSiiQbC z#`_R_R48Fq(&fbTUZX~{m6JX{$Fey;!7z0$_?}p&C%-IEVee2mLMZUHnarBFQ9QPw z&LZW=^jn1g>f{4%^KMPo8?VUddKAPKFvL3HvhVCvLSmr}VDJriEll9>)~2+9m(Z(S zM+RagE@44SLvp9_9vU8v;A2pXrx1RR>F#*`Gq%ZILm|uJ{MtplYB#p&31HiH3Aj+>X`kKQEMxd;%AqIi2DNf>8U^dr z2ywG7ROi`nOPVC?mgh4(e)?we`XgN=LF#3svcc;mHRH2f$LB)@5rotgCqS<+5o<-q zC=UsG&7Sw1agEo?dUWSxe5vp0`#bA5Ff1Y4 z)vaL7TRiSHfsSVBe`DT96Vw00scw1#WDGJCYCu36_RFv-n?D}jkIXcPxovTz=wvVO zR;{(K)N^2JygQPg9*Lvm zqT`&wiqN6qAOeT}hqBx*IYFAsnAV#X?(ThFO*_dV+TvUjK*|tn4TE zB}xAz%da=j_`7+od!B@rd@ph&>+`31fO2S9;&RgH4h?Ii2IH?>#Z=Z;=84>XF*_b- z(d;NPNk43Med{tQIr;dPCyUCV@M5$FzR0*~rJ!_FbgJX4S|vh@Fn`02ldN4r0W%uX zy)VQMpW1KIYw944TrgcSbYoknf85==(I9-q&=dN-?U zJ*r|JXhw0S;AVgXxCIZO*(+Nwrg?S1laMx2;?|%Wum&xiBRi~uel>pFnVG1HxY#l) z4k%()Dt(vF$`uocgqh{Lp{MJow(KdJ)c)OfaX#R%lP!~`Cwi$rt^8o)?pCaozTb05 zce+b=2!_3^r-sww*mwNiZjp=Q?K&aODXH){$t1m4<<_A3uNLezyeHgZ&uq-7Jdm=A zN;h;yR|Ty-WFvZ1lvcQh5)@-2+Iy?dK>39SkMF&xpM`1FTv{H$BcWpOLP2Ny#^J*& zXLbnU;E}!-L-tUxc!*JL!;S11-Y{;FCVZLvDwkWxU5!9GnhXwZq+uEStD&w|H;?&Z zX5u#t!Mdqr?Q!#wq>xFR5c(6zn`h3Q{w1!kg(V2Ej5NMlW_DhMdQdIHUfYE3iCX`y zKB2NV7weBmEXYS8 zdJA?)=XB>Hm+S0&FF4srh*$`ZLbtWB-sGq_J{aB!+hxop!95{dFi)11{;p3%@Z)en z3Iv|Ba@I?Z5FH$3UO1QChbkOzBySjRyx(yX4ZoXnfC~gc@(4gYEJWEF>zKRTC@+1bZ7YtlcX+>%>^m}=B^iCx;R|H zkCSd9)rBie+kqT~2^&#vYPsG4Y6>FWk4;p8M!Q1XOrbA$6F*UQ`N2p{?FH#?erNG9 zYD1KV05da%^`dBa4z6=uQuBHj%5(q04taP<1uE@6AmzE;Hf-k)h}j8D4rUJWvBr9} z@47bmmG$;B7XXsL78k6XOz5t8Pjuc2b0?0L^ATtu&EtHS=|rMJ?!CK^u^ZoVmHe*J z(<_&+qBdhJCz5CH(c_y;2tmY}g+rq2`Xz@AukCsrkBJ>CskR~oi|o_Yz+kggb-x&v zaHX3ESUf}$B!0(CM;T%l>{dIcj7YNeVO?n(mID02ouW?RlkXTux2`D6bn@q`9fG~+ z?~H9_lv04Uu=mcE%K`+*Zjd~6|MC}srOz_2fg8bxQ3K1zX%BJSDhPf;iVqW{tzR!j zx8RH79q5C;rxa4VQh8AJxfN(|Q+S7MicUsi$i9P>`Y90R%ze&x{@(e$;)9secm;%w zU?4_dn+_E8`y$&uNHm#Gnuy$EIHx%!IS&3-+n*?!oic!PVUN?6ZaQl@^4=9 z-=f56#O&)`KQqdTXy*u+0td#+Ub9@(5~o~#w8izcS(jqq;)_{^X;bcQyp;Y3tMV?8 zNq<}pfN(!UmqgCDv!lj<$v#Q4MuoGJVqbZiRx9su(&U{1osvBqjCQO49*K(ATfU&g zsrGh{4=mO5Q5JJM&V#EvofEb{)BR+_d&sOKG{Wf!+pc*>h@BZNJ6DmokS=}ByZ*f? z!Me8`AaPtmmoOIr-0LRYG~hwDdiKVyJa69{yws(Ssf%v^9CrsqC zOW~T^?7T4gNgGFfUu>BZQP_#`P-2~o>SA7qtRCxy3ppO}lkN3~g4nNWCP1ycuRJru=dKA zen)kjds_dBA{{N8`koLsZNI(VI|yd4151)#&KV-jroHVIn?*dtR-^tnUIeP`oFO02`(2kI#u3l`ZD3yc_P)QfO}&x0bFqE+G;bqr zbj_xS)UM)8V!tGV{P|obtSE71E#vorLu|N?@2?es)q3B zQj9e`25}~9mngU3PLcR4fvQHC*2q+P(C@S9y7r{?ijgbT?JqkQ>=M4z^gZ$NdpN;;w7l)g8g3|6wLLCV>8B8V ztBau)oV%NP=seuAIDA#bm#|ed+Wo6#8CVyDKMJ#NqR{}{gmfG&6}7DxaKG_l`+aFV zEY0Rp6%1?>v_4ZGtDQAYQaLTYR#D|+)3oal^|}tXk#frS>OZjymc;PC4=qNzVwmO2 zvq0!KZ-X5~)@I?%h_sia24VU0jC+!U79Kj2c^lwEu0m>2IzIc!=a~s?b3qWdi)iga z{E}_CwVb~yA17FEN^DLv%+JtZHM2b8l(i^oEh{-Qp7#L2bN=zr6du#~%6er30|P(1K)+Le8Pyn)lh zweg2eUD=#?V4EcQDVf;IGLwVR!WugT+Aq#oirVI))q3t7p8@<|2wJQ>Kdu~NXeXv- z0cyYKyWTyBO!nVL=d22uTlkVTN3>y{!Xct3{|za{d6En^5otgyme4xPiCJ3McQua_ zGdXm|W@Q8Cod}|&U%n67jyV#HyEq}+a@)+&z*iV0tIx$>89LVRQoCJehWBz(DN4-Z zZ?Ry~S8oiFd={L4cLU;D`{CzNxaaVuNXk^NfG96(`eT zFt+;RaFM9T!UIZG=fd{WxB0LH2_c{EXK3Pk#X5d#uDes_@~YPJ4A_5`GG@KWU-r_U z?GUT9o$e;gue2AFDXF2TG|Kwa>e@C0J>EkP&6Aw02a9hNYV|7@CFp2dwq4d; z3{4}A2xNFHS&B_wFgnUMxS6qR=JoLpIEd4&;-rrbvi3yK0?t1dPhkl@0OC^rpob}^ zWwZyY(GEDoRwKkQ=gHd;GIF6AA!GUCu?RL&DtH|dY_iCB`m_eg`Ha<(pnvt% zHd!#Fv8CC~aotgfHfY^)I6a}tS6&Q8<#zlRX9FhNAx&Y*-#QwR zigLF_REY-L#d3!Z{i@0uejKU&w@D0iEkU2BcyX4&*>!IO;5ACshzu2eb zTJ3CmTLH$V%Y14XsIjG6c8OkJT63CcQVU`~iC26Lb}hht)m49`qX?rXX{41ZZQHl4 zLu_%X2>2o=_aGwOH|Q;>V4-eGQG8=>R1*i^C(ElwVCE`1@yM}ajhgadl|YeY@Kitd zUCK8*ctfoQrScXvnMctE^6d6a$j!?tK~D$>_S8WBp5jD1fmEQw@*&601Ub?9_>1RV zh@`)~?Mf7rltJr|C*PiINcxzpYStb36+~lbq^xdkn@B65f(|-qjRli%fy1Qt&y{_uRi5%aLsnnpZCW1-<*B zq%OL=dX?=wPU(>Ny#$aDqeLx)rR2g-nRa!iR+XBeFY7J+C#1STcruA5fjV)5zyllu zMr5*`TGd9j&2sry)>lql?%Ja;v5(yzAvSySjxwhPv9m=-b^y2^C852tjFZp#CbCu0 zJ5km~eW+-WXGIdOw$^$J0g%6yayH7QV*tW)P5$fvO(*-D`risCDfQyRB7##lcQ{rH z6=jp1H>J~MC^A0hlBF3z5$JPKN^ud0vEj;N?1Qm6*>7ZPYaGVV)13ej&AK~C73FZ=M^)Pn}E*w zgT1P{H+5PlL=G<^5!1I09+``mec-D%ArgCL*vMZK-+Wt(qEJ8$=6PooQRA6XR@SyzjgG}Tr|4#h$ zC*yPCNe50b;VFA}`Q;E!0jQ+ZD$}f$;kQh44P0Z}hz?4+ND7}Xj1_WICDP;%?Ms@I zryWiy?1Ok}pGl0B#m|=(>1M560GGoi7s635PKh;)ip__@9I}naRhLhdF}ZOlM8#sL z3(yn0%r80KP(Dz88yr9U3xayU`g#IOENc@@Uy;?UQSrNVz2onTakwV|xz3UogCXsM z5CGl;+q?4cP$ch%P#AG5OqI zgU{qGjDIAbqwv)kv;Y^$$B7QxZceDb& zMcdzJnY9x3lDp%l2W=V8yvVLA<1T7i*9|F$RL_or9=+QO8ssR8Q@DsS$qHUD2`Yr@ zVN2YWuh34LGbKIk=aPSh&{Hl+MEFUaGgOgzcU6}oFDD?JGvo$@$iOe(v2Zs-8tRP8 z3>?1L6(s$ILcdKo{h$WqJcdPHg)sZ}4y0$gf%a^dpv-(2|AlG}z~ZQrOkEJ2>=O7K z5Zw>}a`hLLH&>Qn4X(sf<=hj5U-e=;WxEzg;yK=$8tW;c|BTla7rtbAoM3-tBqQ`t z?+CGg7(n=dvJlTN@8Gb5r;(fJL@Vs-PT$c?#S-$kP;baGM$)&U70H`nJ;8LsO2EF# z-=Az5L%zYDF2$!RYf)6ql2LdylqGbafkZU>A~4Q}lLJ!WC8fp;oRHM$bZcI5fEb#L zTebE}?Uxi!u*=lDM#&trZm3lLd>Hq{8``zlxaBmmhU~DXs0#yQgWzowh#o>|iKaSf z4f!Q6SobL#;uY;U>`l>i{rimREfiNNouB%-O>&)}J`TJo7mN0uxnddfIT~oX%rQx* z_t`l8C6Kd5L^nbxQKYx)76yry_qD1_y5W4x!+wp@=OqI!2=Ny@I@yXAA_ZbR+!$@n zNKT>0LDee}iSm8SGwy<&cn#YH5jV2Nntq(hYzv(zu_&!ohM!gwE1v~~5%%ma%}<<} z!f8LzDD?S3w{B>Bj?KX6z}?9&pM6+Pjk?^?@9m0US@s3qwvg&=aLOFU-%4p$Xz&n5 zx0%Ytk4Q6Igr*Y>%OT5-3q=2NpcPI4Mx5mji2EACLZ~-#5-drq*T_vABNLKTQZyPp z)3}!vt`qBY{e#O`e-=>I>cax<;wu@(luXetxOm-Nz{()=vE_ zFQTWTP!hg#1ZitZaNd3z{AQH$fO~4IqG*X(gQ-7sAcnbsEyN57Ku?AU^?^QL4S01S zV6oeijVHPLK{+10^m>R(>M!kmrFH3L^g!Y)g%I*Bfr4Nag?DJ>A>T_UB|~NY;90N* z8a-%)i>Aap=X`=1AyU)f7K1XFSmBbAN6M!gL42`bk%!++L``gt^wLx0LiCUT@(@ud zAaFD=&%M;LAOU4@{%ljOJ1e=7U;1u`Hgo zZY%?)sw`+4k|df0zJAF&fleT}JhM4CdR7OYjF(zl)kxib=d<`xl7w7f+uw_NfR$($ z5DbxzL(4R&kqb)H@}=;WI+IX@a!}jIG}f{~ZqtE-VKq2?bs6RZB+;^1T&xh1LM0A5 zsb=&V2mRlsy9PKb`)t9lgO0u7VvE$t?XEmY&ZaMgf1t}~_;*Pwj3uYDf#*k5MC4r$ zzkyXD9+jokR8Jn`nL(xpUme;L*bfAT~!%VfwdRm;8kZ|DsD6AIGG~nEf>#vuXGs9vQR3{jTdvd087*nFkl&P|EGP&L=t`8bS@>W(>Hm!85G8BW=q?n`nYS|u>b#iVjZ^PVI_ z6SkP_5x-W3)tf5b3a-^6v0~6~*=r9es;rv^E_>&CqKgoQ3Ce}gvxGJ)5jy=w4?x|O zZ?7V!94ldfeBIldC=4}impn%`l}4K_CL;2s>3V8)Pr?QzFs{!~{O+Vf@O@z0^tQ{+ z^3mhN{h67~+WDS@=k9drT;~?Kft&rF8WVi(QWZf9ENiX}r*^9MSdK z;#-OM^V|DfKR5Zc2Um&qd8_^+9o;vsK}RB@R(}nL-9iK6^7Uf=qt5VSTNkfZ{U*dZ+zztRB9XVl&33j?|LB}_&WU*DgJuzmie@9k5Z+Zgz59Q_{!&8$E- zcERFN6oyQh6;+AC~Q$ZaJbxm)b1Gz=U?A zlG^1@(*27asw+pN+9r#wOZa=&`z;y7hVC1ApBsAQOag=CGg8ljs7Dy9PKFv;;`!II z#C^M|Kj?s=9LR&~yRNYtrskF~7#Jd(e_DY2?+GFrU7q&7<(7$E^JeokE*>|jJnH1$ zf5{|nG1!jEs_$3z&7M#De>$b{JW<`fJ_l(3eUL%@IJng4-fVF(RJqz>zH`ab-n!!D zc{kQxB6jvMiz`>BwScpB>5OR9k9%i{zUvde+uCjA*uBA4%zu3J_P`G8ncXxCmplWm zK>>Qde_yu_jDNN&{YA~dZz2KxVR^~7pAKtNX5E`UOq^?M;?4XN>}mR1JUb?uPwfVA{${Z9M*Y+9ix|FCF}3NzXPV5ZAS2zLiwT zYv6zDYdiYbo~MsrJr=X)L>M$F1|{}W*AWPTO);}3&fn$Y_n0`QBsBex#w*|S|Iv9$ z+@1*X$W2a^lSBzzs_z*MiiA)~A3aVbYhTHs_h|%jsPYP-xk-U4mzQ~QV^QBcsC9}& z9Qp;w_qAF`@&X&AhT=v3)dBSXV3Up-AH_>G!tHU)7M-GZ)Gc@?@T~nG^`06ET(53& zh^83!bF;k8T|HO$pA;Yz_`bMvx&_Qr?sTthRxID>ePgi;;{t;@xn!_-iUT*u}g{Ex@K<^jdq>2 zWb%gsvj52lQbRTWcHEMa3#3!q0|Cno+keo)9{O{%m{uhlBeLD){wpGtbYQu*OaIIq zC>6fqSbZJ}g1gnYMgM5V(dct9GXaGJ_%ymK4eH{v{%IcwJaZ)~@{5PXNCPcwO)jSo zYIk$ppR|!h_BYV6s%D=g3M6(8mu5qo@$=V4gdMt>q{xfup*^Y*+n;P>CNlf5YbF1w zKoiSI%;589x6OF7(}jr{u2LqGDy}?1Dj^V1!jb7~ zl&~;gIyzR@7KRolL z{|b!(dacva$T;sU-1`>pYqP~pl}z}8l~_`c`r##EhGMx_>meDGiw}@j+$CsacSBp8 zN@f&X3>l3{p&tQ(@!OHHk z@5orTf3)H1V)|#bTf#cQp220uIW*BD`2@7Wh{6RVf5>wEu63KII=E&{dFh)0=LCNR z0?e3Bb4Kzg=Q=_h0kP^%+fK_^KI&o5lsfc5dkGNJ?H92)Z^@+Q{cwMIQJS-b(Lpy0 z%Bm3arHVS!Zjq@k5EckebhtqWD1qpnBokH)`I+4NYxghAYlMzihKHoyfIh8p%ii08 zVC3W|O}zc^XVqI7TN^Ayq^(!qIoLCpO4<;yaom-XGeA}njaNeY3ckEMBo`X3zb^oSIZZ51*h;>qKV$}=s&OHxkh72-+NToOA!&o{{-9uZw`DgR=MI@QYWO*kI`nQw2;v}kg1<(X*dUk* z6bj2lY#rZ^fy9YC_Cb64iyn-Mv}?#({C7z?R5`LH0^Lx*Q2&lOI_QvpJ(zUe_wg8h!TNrvDBmT>0FfSBLHRTHVSm;q~BXldyuiK}GIVqyJ0#Iqp;|ed)6F*nE#M^?C#4oId3NIo2DostZC*8r2;X$bLl5N5+l(0eA1F zQn~Spa;%e@L$a>Z`}V`b1>C6?ypm8UiGMnBnT606Ni(Os9kg1USm(njqaEto0D28%sl-6^8U_+DS1aV0b-AdM^oZtWl0CWO6^;&4JCYt!L`CN&SN91h6_ST;9m z3|ft*WhX-<`u79H0(U!P!8I3ru#8k8Q&I)8zo@_P6^L>nvqjjyOkX zL(TBQcJTLG3%u3t!Y{O@dYA@gC3`+kaC=BA;1#)vMNO)K@aq^BB+Z3f17R$2DHv}z zBbrQuCHV|%SP!6-RZvGp7FBH`<9L${zT852tcpHfn}%2o&ZZ^VYu!o_cz1lbM_I0f z@1zxG1w;V3=|~@>mLq$khr`{4&g)Sebm2yj(rM53(7vOm$n%GvpX~%(wgnjuUKqoj z%FO(hkT!=M2E{0-Gk865o0}T3e#I^9wb=(iwUhEQXJiaIM;6;T)};A%exBpTqAv55}-tc%5vN#7o_`^MFou;lq-f~&f1$nL)1njB8p2;;&j2#-w*>Ivk% zyOI4gRM8y(mzzZK#mk)Tu-g)dF`DmCAq$2zfjO0WSNbz~CEq`{y+Ftfc?0|0iq@y- zK_Xm~mU^FMQ0DuWe5tX;K;rJ*8$h0M2SvOJmf=9ILLeK_9Pno|L}!qTv5+@Knn1wn zreCbz9&7Gx3yQ?H=3P-Zc#Dp^h~OJ%0$DXPX{aCR&hY1W!qnUov3cY(0q178lxLAI zHdP=zVI!QV&Bu3lxRW8wZ~-DTrS*cf0mv?3YSBd_WD3IWS$I??ZP0kn>R-?+NAJ-S z>qgGOHjdfcbUMX>(16QfWvlaAi8yC(lN8(aGfgjvMowuSWDm^&nui8|zfA-n-j%nJ zt-G&_Mh|-E&cZj+8`LAyJZ5reB(z%Jtql77W~+pmi?9xIk8bE&huvnMBb;MpD394du@yg>Jf#~+9#*xqOyb2;|cF;~?XVw)oMi3jz+)H}+`uJvrTw3EBbJu(fvax}AD zL&MlRn%;S3^;wXcN5m~>ezMPan~8W6Uk1|BU$$y_gvf_aEiNJ7ZRIE!eK| zhux}!-~4pn<6dk0e(fawMi`67Lxop-&{FQF8s5olT}r91}JAm<+@bi$_Xz%8#^%bL!8-8Y$EeSM%}0pecW&Nev

OB}#(hK&ZWj3s`n{7HrqjtxUSa`@>~z?#eOju(Nt1z7pe-=|v*U=Ekv- zvB>rJh)W)ALCqQR(daQ*!uj&;)>8ua32{)Bh?V|;Di~W#_Yw|bx5d{m*`?_=2jD2t z`SJ?v`+8pBDQ?^7(F}h|ql35_SF4RIz}n1;bo;S<2~S||*ug7PS2x`L^I#c}?Z>#} zkSMmzuWX2<9sEv{}ruJgU^VCR&2SE z;p;lUs6OB)Gwv+O-1kV$020>bKYQ^r5v(bd@qa0TKFB_9WdibISQ%MxH#Y+NH_0lf z87r3bDWM+hLjz<&dPC{bB^Sq+V7J|n{q%zpNx#PP3mNbO#ujUNGIyeTC$xjji_Kao zqXX@VI?Y5sW7@VO1SCZ)q@ieOFrv1FM|T@ax$wfWqUvmY(rDP+c%|_wcc*3?2C;HJ zOvUW$%=Az$rRhJ0;)x$Qa|P`_b|~ZVLn@PVZg~Ja@zUt~@Hrt+RVeN{?3qY|c<}wW zVgQ5s2%?T+!!Ks^$+d2jlaDEK6NEM#RoyP}0TfG&AXR|NWlwV3;&iY9%kr#~*QZ*Y z7#M8tfocarx&cDLiYb4S_=!la4Jsc!r6E40jeF>DpatoJD74|Vf-C92&g4SL=ci}^ zuLgcq+0@K)lslTv@O%6mBPO^adFvk1#1m~ebhsIL>X=p$J(dg9lm=I< z3G$e>0KU*b{&fIqLX@>h3)STrmLXNHuv(L?*Qp}kp7IKagPGVk2IybtvF_WOf;5&m zm5c|(KZkLKb0*j*!S4$YE3AVP+xCB_0y;VZ@g_)fBYbmd?Co^Mo@tt`Hj;V8zQf+y z!i}$F$ne(&S4NRQbB=QKZJ%k-@==ArfMkeMgtbX@j%#6pM*fI6zbrK�+w#YfYu9x==Sog@SmS=U`POT9?AfMG-ESqPBdf{8;3K+yCxUZ< zbK8c_uOYeCk0Dc3F7j(q+P!hfz)&tw=kbDx%ebgrlnO%y)4YJXsQK+1#i^}44Pzcb zA?;m+wO#NMh*^iY8Wg}%(bPi~y?d$^p7j0AHM)UUCb67(X#xuuxeBpGn|9}`guLaN z)Y1#z_m`kJNIxM&93Z>i1*!NTx&FcY$d|4rjlwf0>5rt*F4;i!Yx4yaGNK-qaA?)8mN| zME*?9{c)GPk{$sul_jd=)!g>wlHILk-PYBpMs~)hvwRaNaJ;5Ae}9X+G%M+deHChJo#HOHA3&mN7)~BBMP~Ceh%QKfprNg;N z$fpAGSW-ANT#=nwHZF$?x)mCRgGd``l_pWOt~?0~4ha<|k7h^w03S{9yvkGY7I*=c&@fz&@Sl@2aZQ_LRhM?C9@@)TwUKZQ%hq|y}Ac|fp1 zt}^PxvpFZ3Nz5?K?=g!7??3VFZudzh6NGVJS}57zTgvH=Yi?)mkfCL~w+LCI$#0ad zp+Ie76R?A7GpdQ_S}J4lSO)0S=w$ADFLo<3f-7+xVj8w^^=hOS41?NL9Df85e@dq9 zWJ3+b(8w4h58w7tcv#3rr^r2dlNpc*MSC9%zKNH4HyH8CULhA68(TaNu9Gqh|4BV= z44?cdaVr=%9AEmk3|ygV zXtj=LVmo(e^&7govvs3HX!Vg^$kI`gFS?7s`!0&u?W{c5%0Gs+~&!W@mLl~#>Tj)JHXIc*8H8umI0xMOeP_%FT>jOWN|Evn*g z<2d_^wxq$(cFZ=prmRp-a5At&1}Aq?9JjeTk%M1)DojHDI1v6DOq=|Bxa`yhIW$~O z5>$%tPpV`erbUJAK%dv$2EQGZ4wjVD#(@eJ9yCMzcCvh1=qk-jwn=QLU561ZaY*|~ z0jEzG;C;F3d`2bmA`;$@$=@uEK>cXSOy;H&A>(YV=E=yDO6DHFm=y~#zV+V9O02Qf z%=J^8U~FPs84=EI7{js<+iYLWBQD_G-up{;V7;(Ix3Vy!nhH7YB_EEY3aDqgsmf{S zV^_es=F8hg`~y}}X;@@irYz(h0T{QeN(B>y#RUvsX(2oWd%V6jo?^tSF;h6utvqru zv0EU*HUD)5KnZ|W2oLqW>*!yx>JOeQ&;yFLA9cY1FOne;+~${@{dWu{ z+W$5k;VEb()@^(ai~hWf=#M_RJ<7;Eg+Dp=N=QeCr!8#lO2G9=Ln)UzRhB@GMZzLF z{mM8Yume%!lYY{6;xAkf=oG+dnYxVY5)`YM0writXXOy9pm!ra_#6@y%)#;}tg$JQ zH~gOD${4Il*|Pt@s0UZJb7|=4Qquf~F)2y3q4*0r(i1Q$Ld^*nnVsnCZO}gh4K-f94z;@N@KPpW@8fpw2P&OokJyp!l_89P8{8|3j>pBR09$ zrGUJBZg~>te9jb_#|Ho0RLZLWB#)or>On9jMaIb($nK&(ylu8I$k~YR$7^XQC2>Ny zfiRo*NamGl(HKCHCnw)Fo*yE literal 0 HcmV?d00001 From 1b2fcf5a98f36dc74a236b589ad79e0dcd109587 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Fri, 31 May 2019 08:16:51 -0700 Subject: [PATCH 06/40] Add author lines --- conf/czbiohub.config | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/conf/czbiohub.config b/conf/czbiohub.config index ff1523e..1c788b9 100644 --- a/conf/czbiohub.config +++ b/conf/czbiohub.config @@ -1,12 +1,19 @@ /* * ------------------------------------------------- - * Nextflow config file for UPPMAX (milou / irma) + * Nextflow config file for Chan Zuckerberg Biohub * ------------------------------------------------- * Defines reference genomes, using iGenome paths * Imported under the default 'standard' Nextflow * profile in nextflow.config */ + //Profile config names for nf-core/configs + params { + config_profile_description = 'Chan Zuckerberg Biohub AWS Batch profile provided by nf-core/configs.' + config_profile_contact = 'Olga Botvinnik (@olgabot)' + config_profile_url = 'https://www.czbiohub.org/' + } + docker { enabled = true } From 9c0315addb194e2360c6fe76d53a9947d3866fab Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Fri, 31 May 2019 13:17:48 -0700 Subject: [PATCH 07/40] Use flat nongzipped files --- conf/czbiohub.config | 52 ++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/conf/czbiohub.config b/conf/czbiohub.config index 1c788b9..d51c6f2 100644 --- a/conf/czbiohub.config +++ b/conf/czbiohub.config @@ -47,59 +47,59 @@ params { // GENCODE GTF and fasta files genomes { 'GRCh38' { - fasta = "${params.gencode_base}/human/v29/GRCh38.p12.genome.fa.gz" - gtf = "${params.gencode_base}/human/v29/gencode.vM19.annotation.gtf.gz" + fasta = "${params.gencode_base}/human/v29/GRCh38.p12.genome.fa" + gtf = "${params.gencode_base}/human/v29/gencode.vM19.annotation.gtf" } 'GRCm38' { - fasta = "${params.gencode_base}/mouse/vM19/GRCm38.p6.genome.fa.gz" - gtf = "${params.gencode_base}/mouse/vM19/gencode.vM19.annotation.gtf.gz" + fasta = "${params.gencode_base}/mouse/vM19/GRCm38.p6.genome.fa" + gtf = "${params.gencode_base}/mouse/vM19/gencode.vM19.annotation.gtf" } } transgenes { 'ChR2' { - fasta = "${params.transgenes_base}/ChR2/ChR2.fa.gz" - gtf = "${params.transgenes_base}/ChR2/ChR2.gtf.gz" + fasta = "${params.transgenes_base}/ChR2/ChR2.fa" + gtf = "${params.transgenes_base}/ChR2/ChR2.gtf" } 'Cre' { - fasta = "${params.transgenes_base}/Cre/Cre.fa.gz" - gtf = "${params.transgenes_base}/Cre/Cre.gtf.gz" + fasta = "${params.transgenes_base}/Cre/Cre.fa" + gtf = "${params.transgenes_base}/Cre/Cre.gtf" } 'ERCC' { - fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa.gz" - gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf.gz" + fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa" + gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf" } 'GCaMP6m' { - fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa.gz" - gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf.gz" + fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa" + gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf" } 'GFP' { - fasta = "${params.transgenes_base}/Gfp/Gfp.fa.gz" - gtf = "${params.transgenes_base}/Gfp/Gfp.gtf.gz" + fasta = "${params.transgenes_base}/Gfp/Gfp.fa" + gtf = "${params.transgenes_base}/Gfp/Gfp.gtf" } 'NpHR' { - fasta = "${params.transgenes_base}/NpHR/NpHR.fa.gz" - gtf = "${params.transgenes_base}/NpHR/NpHR.gtf.gz" + fasta = "${params.transgenes_base}/NpHR/NpHR.fa" + gtf = "${params.transgenes_base}/NpHR/NpHR.gtf" } 'RCaMP' { - fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa.gz" - gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf.gz" + fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa" + gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf" } 'RGECO' { - fasta = "${params.transgenes_base}/RGECO/RGECO.fa.gz" - gtf = "${params.transgenes_base}/RGECO/RGECO.gtf.gz" + fasta = "${params.transgenes_base}/RGECO/RGECO.fa" + gtf = "${params.transgenes_base}/RGECO/RGECO.gtf" } 'Tdtom' { - fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa.gz" - gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf.gz" + fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa" + gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf" } 'Car-T' { - fasta = "${params.transgenes_base}/car-t/car-t.fa.gz" - gtf = "${params.transgenes_base}/car-t/car-t.gtf.gz" + fasta = "${params.transgenes_base}/car-t/car-t.fa" + gtf = "${params.transgenes_base}/car-t/car-t.gtf" } 'zsGreen' { - fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa.gz" - gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf.gz" + fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa" + gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf" } } } From 4491907f1461185b8be7e6906e9d7b583dc91fd3 Mon Sep 17 00:00:00 2001 From: Tobias Schraink Date: Tue, 4 Jun 2019 12:56:11 -0400 Subject: [PATCH 08/40] updated singularity version in conf/prince.config to reflect version installed on prince The prince admins changed the singularity version to 3.2.1 from 3.1.0, making the old version unavailable. To keep this config functional, I updated the version number for singularity. --- conf/prince.config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conf/prince.config b/conf/prince.config index 6ae4ee8..1c2ea2c 100644 --- a/conf/prince.config +++ b/conf/prince.config @@ -1,4 +1,6 @@ singularityDir = "$SCRATCH/singularity_images_nextflow" +singularityModule = "singularity/3.2.1" +squashfsModule = "squashfs/4.3" params { config_profile_description = """ @@ -17,8 +19,8 @@ singularity { process { beforeScript = """ - module load singularity/3.1.0 - module load squashfs/4.3 + module load $singularityModule + module load $squashfsModule """ .stripIndent() executor = 'slurm' From 3a481402b82debaee410afeb2b64d7ad5afd1039 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 5 Jun 2019 14:42:07 -0700 Subject: [PATCH 09/40] Get czbiohub config to work --- conf/{czbiohub.config => czbiohub_aws.config} | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) rename conf/{czbiohub.config => czbiohub_aws.config} (95%) diff --git a/conf/czbiohub.config b/conf/czbiohub_aws.config similarity index 95% rename from conf/czbiohub.config rename to conf/czbiohub_aws.config index d51c6f2..baf4550 100644 --- a/conf/czbiohub.config +++ b/conf/czbiohub_aws.config @@ -20,8 +20,15 @@ docker { process { executor = 'awsbatch' + queue = 'nextflow' } +workDir = "s3://czb-nextflow/work" + +aws.region = 'us-west-2' +executor.awscli = '/home/ec2-user/miniconda/bin/aws' +params.tracedir = './' + params { saveReference = true @@ -42,7 +49,6 @@ params { // AWS configurations awsregion = "us-west-2" awsqueue = "nextflow" - workDir = "s3://czb-nextflow/rnaseq/" // GENCODE GTF and fasta files genomes { From 268e411d10eefff821625e0f72346d52cf30842e Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 5 Jun 2019 14:42:15 -0700 Subject: [PATCH 10/40] Add czbiohub_aws to list --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 7374fa0..0ca7fbe 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -16,6 +16,7 @@ profiles { ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } + czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From ad7e26ea69388a906397c41da9363a91c76c2a1c Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 5 Jun 2019 14:42:24 -0700 Subject: [PATCH 11/40] Write out docs for czbiohub" git push --- docs/czbiohub.md | 141 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 docs/czbiohub.md diff --git a/docs/czbiohub.md b/docs/czbiohub.md new file mode 100644 index 0000000..6234fc2 --- /dev/null +++ b/docs/czbiohub.md @@ -0,0 +1,141 @@ +# nf-core/configs: CZ Biohub Configuration + +All nf-core pipelines have been successfully configured for use on the AWS Batch at the Chan Zuckerberg Biohub here. + +To use, run the pipeline with `-profile czbiohub_aws`. This will download and launch the [`czbiohub_aws.config`](../conf/czbiohub_aws.config) which has been pre-configured with a setup suitable for the AWS Batch. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Ask Olga (olga.botvinnik@czbiohub.org) if you have any questions! + +## Run the pipeline from a small AWS EC2 Instance + +The pipeline will monitor and submit jobs to AWS Batch on your behalf. To ensure that the pipeline is successful, it will need to be run from a computer that has constant internet connection. Unfortunately for us, Biohub has spotty WiFi and even for short pipelines, it is highly recommended to run them from AWS. Make sure you have [aegea](https://github.com/czbiohub/codonboarding/blob/master/guides/aegea.md) installed to make launching AWS instances from the command line much easier. + +### 1. Launch the instance + +There is an Elastic Compute Cluster (EC2) Amazon machine image (AMI) set up with everything you need for Nextflow already installed. Its ID is `ami-091ec599f8e77734d` and can be launched from the command line with aegea like this: + +``` +aegea launch --iam-role S3fromEC2 -t t2.small --ami ami-091ec599f8e77734d --subnet subnet-672e832e $USER-nextflow +``` + +For example: + +``` +aegea launch --iam-role S3fromEC2 -t t2.small --ami ami-0adcc973d6f458a1e --subnet subnet-672e832e olgabot-nextflow +``` + +### 2. Log into the instance + +Log into the instance with `aegea ssh`: + +``` +aegea ssh ec2-user@$USER-nextflow +``` + +For a concrete example: + +``` +aegea ssh ec2-user@olgabot-nextflow +``` + +### 3. Start tmux + +[tmux](https://hackernoon.com/a-gentle-introduction-to-tmux-8d784c404340) is a "Terminal Multiplexer" that allows for commands to continue running even when you have closed your laptop and lost your connection. Start a new tmux session with `tmux new` + +``` +tmux new +``` + +Now you can run pipelines with abandon! + +### 4. Make a GitHub repo for your workflows (optional :) + +To make sharing your pipelines and commands easy between your teammates, it's best to share code in a GitHub repository. One way is to store the commands in a Makefile ([example](https://github.com/czbiohub/kh-workflows/blob/master/nf-kmer-similarity/Makefile)) which can contain multiple `nextflow run` commands so that you don't need to remember the S3 bucket or output directory for every single one. [Makefiles](https://kbroman.org/minimal_make/) are broadly used in the software community for running many complex commands. Makefiles can have a lot of dependencies and be confusing, so we're only going to write *simple* Makefiles. + +``` +rnaseq: + nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ + +human_mouse_zebrafish: + nextflow run czbiohub/nf-kmer-similarity -latest -profile aws \ + --samples s3://kmer-hashing/hematopoeisis/smartseq2/human_mouse_zebrafish/samples.csv + + +merkin2012_aws: + nextflow run czbiohub/nf-kmer-similarity -latest --sra "SRP016501" \ + -r olgabot/support-csv-directory-or-sra \ + -profile aws +``` + +In this example, one would run the `rnaseq` rule and the nextflow command beneath it with: + +``` +make rnaseq +``` + +If one wanted to run a different command, e.g. `human_mouse_zebrafish`, they would specify that command instead. For example: + +``` +make human_mouse_zebrafish +``` + +Makefiles are a very useful way of storing longer commands with short mnemonic words. + + +Once you [create a new repository](https://github.com/organizations/czbiohub/repositories/new) (best to initialize with a `.gitignore`, license - MIT and `README`), clone that repository to your EC2 instance. For example, if the repository is called `kh-workflows`, this is what the command would look like: + +``` +git clone https://github.com/czbiohub/kh-workflows +``` + +Now both create and edit a `Makefile`: + +``` +cd +nano Makefile +``` + +Write your rule with a colon after it, and on the next line must be a **tab**, not spaces. Once you're done, exit the program (the `^` command shown in nano means "Control"), write the file, add it to git, commit it, and push it up to GitHub. + + +``` +git add Makefile +git commit -m "Added makefile" +git push origin master +``` + + +### 5. Run your workflow!! + +Remember to specify `-profile czbiohub_aws` to grab the CZ Biohub-specific AWS configurations, and an `--outdir` with an AWS S3 bucket so you don't run out of space on your small AMI + +``` +nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ +``` + +### 6. If you lose connection, how do you re-attach the tmux session? + +If you close your laptop, get onto the train, or lose WiFi connection, you may lose connection to your AWS EC2 instance. To reattach, use the command `tmux attach` and you should see your Nextflow output! + +``` +tmux attach +``` + + +## iGenomes specific configuration + +A local copy of the iGenomes resource has been made available on `s3://czbiohub-reference` (in `us-west-2` region) so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline. +You can do this by simply using the `--genome ` parameter. + +For Human and Mouse, we use [GENCODE](https://www.gencodegenes.org/) gene annotations. This doesn't change how you would specify the genome name, only that the pipelines run with the `czbiohub_aws` profile would be with GENCODE rather than iGenomes. + + +>NB: You will need an account to use the HPC cluster on PROFILE CLUSTER in order to run the pipeline. If in doubt contact IT. + +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From c7eed3fa4c92d92c8b5ef442d1b18df1256f4529 Mon Sep 17 00:00:00 2001 From: maxibor Date: Fri, 14 Jun 2019 19:16:20 +0200 Subject: [PATCH 12/40] fix queuesize scope and singularity --- conf/shh.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/shh.config b/conf/shh.config index 2b195a6..5cb3396 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -8,12 +8,16 @@ params { singularity { enabled = true autoMounts = true + runOptions = '-B /run/shm:/run/shm' cacheDir = "/projects1/singularity_scratch/cache/" } process { executor = 'slurm' queue = 'short' +} + +executor { queueSize = 16 } From f40ed194e3afb9d0c6ad0c511943c404ad690cd9 Mon Sep 17 00:00:00 2001 From: Szilveszter Juhos Date: Mon, 24 Jun 2019 09:54:37 +0200 Subject: [PATCH 13/40] Using correct reference location for munin --- conf/munin.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/munin.config b/conf/munin.config index 5d95f01..bca56f7 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -28,5 +28,5 @@ params { max_cpus = 16 max_time = 72.h // illumina iGenomes reference file paths on UPPMAX - igenomes_base = '/data0/btb/references/igenomes/' + igenomes_base = '/data1/references/igenomes/' } From acad9ac7e5eb850361a4797000199641ef29b3e9 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 24 Jun 2019 17:50:36 -0700 Subject: [PATCH 14/40] Try to clean up czb update --- conf/czbiohub_aws.config | 123 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 conf/czbiohub_aws.config diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config new file mode 100644 index 0000000..5c47c12 --- /dev/null +++ b/conf/czbiohub_aws.config @@ -0,0 +1,123 @@ +/* + * ------------------------------------------------- + * Nextflow config file for Chan Zuckerberg Biohub + * ------------------------------------------------- + * Defines reference genomes, using iGenome paths + * Imported under the default 'standard' Nextflow + * profile in nextflow.config + */ + + //Profile config names for nf-core/configs + params { + config_profile_description = 'Chan Zuckerberg Biohub AWS Batch profile provided by nf-core/configs.' + config_profile_contact = 'Olga Botvinnik (@olgabot)' + config_profile_url = 'https://www.czbiohub.org/' + } + +docker { + enabled = true +} + +process { + executor = 'awsbatch' + queue = 'nextflow' + errorStrategy = 'ignore' +} + +workDir = "s3://czb-nextflow/intermediates/" + +aws.region = 'us-west-2' +executor.awscli = '/home/ec2-user/miniconda/bin/aws' +params.tracedir = './' + +params { + saveReference = true + + // Largest SPOT instances available on AWS: https://ec2instances.info/ + max_memory = 1952.GB + max_cpus = 96 + max_time = 240.h + + seq_center = "czbiohub" + + // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + igenomes_base = "s3://czbiohub-reference/igenomes" + + // GENCODE (human + mouse) reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + gencode_base = "s3://czbiohub-reference/gencode" + transgenes_base = "s3://czbiohub-reference/transgenes" + + // AWS configurations + awsregion = "us-west-2" + awsqueue = "nextflow" + + igenomesIgnore = true + + fc_extra_attributes = 'gene_name' + fc_group_features = 'gene_id' + fc_group_features_type = 'gene_type' + + // GENCODE GTF and fasta files + genomes { + 'GRCh38' { + fasta = "${params.gencode_base}/human/v30/GRCh38.p12.genome.ERCC92.fa" + gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gene_type.gtf" + transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" + star = "${params.gencode_base}/human/v30/STARIndex/" + } + 'GRCm38' { + fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" + gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gene_type.gtf" + transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" + } + } + + transgenes { + 'ChR2' { + fasta = "${params.transgenes_base}/ChR2/ChR2.fa" + gtf = "${params.transgenes_base}/ChR2/ChR2.gtf" + } + 'Cre' { + fasta = "${params.transgenes_base}/Cre/Cre.fa" + gtf = "${params.transgenes_base}/Cre/Cre.gtf" + } + 'ERCC' { + fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa" + gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf" + } + 'GCaMP6m' { + fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa" + gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf" + } + 'GFP' { + fasta = "${params.transgenes_base}/Gfp/Gfp.fa" + gtf = "${params.transgenes_base}/Gfp/Gfp.gtf" + } + 'NpHR' { + fasta = "${params.transgenes_base}/NpHR/NpHR.fa" + gtf = "${params.transgenes_base}/NpHR/NpHR.gtf" + } + 'RCaMP' { + fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa" + gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf" + } + 'RGECO' { + fasta = "${params.transgenes_base}/RGECO/RGECO.fa" + gtf = "${params.transgenes_base}/RGECO/RGECO.gtf" + } + 'Tdtom' { + fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa" + gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf" + } + 'Car-T' { + fasta = "${params.transgenes_base}/car-t/car-t.fa" + gtf = "${params.transgenes_base}/car-t/car-t.gtf" + } + 'zsGreen' { + fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa" + gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf" + } + } +} From b0c67e0a9b45112c570b0a9856a4ed479190ab13 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 24 Jun 2019 17:52:10 -0700 Subject: [PATCH 15/40] Add both seqCenter and seq_center --- conf/czbiohub_aws.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index 5c47c12..af7707a 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -38,7 +38,9 @@ params { max_cpus = 96 max_time = 240.h + // Compatible with multiple versions of rnaseq pipeline seq_center = "czbiohub" + seqCenter = "czbiohub" // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket // No final slash because it's added later From b82d68c009a4deb4c193bd440982381c4ba8556d Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 24 Jun 2019 17:55:01 -0700 Subject: [PATCH 16/40] Add docs --- docs/czbiohub.md | 125 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 docs/czbiohub.md diff --git a/docs/czbiohub.md b/docs/czbiohub.md new file mode 100644 index 0000000..5e98f4f --- /dev/null +++ b/docs/czbiohub.md @@ -0,0 +1,125 @@ +# nf-core/configs: CZ Biohub Configuration + +All nf-core pipelines have been successfully configured for use on the AWS Batch at the Chan Zuckerberg Biohub here. + +To use, run the pipeline with `-profile czbiohub_aws`. This will download and launch the [`czbiohub_aws.config`](../conf/czbiohub_aws.config) which has been pre-configured with a setup suitable for the AWS Batch. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Ask Olga (olga.botvinnik@czbiohub.org) if you have any questions! + +## Run the pipeline from a small AWS EC2 Instance + +The pipeline will monitor and submit jobs to AWS Batch on your behalf. To ensure that the pipeline is successful, it will need to be run from a computer that has constant internet connection. Unfortunately for us, Biohub has spotty WiFi and even for short pipelines, it is highly recommended to run them from AWS. + +### 1. Start tmux + +[tmux](https://hackernoon.com/a-gentle-introduction-to-tmux-8d784c404340) is a "Terminal Multiplexer" that allows for commands to continue running even when you have closed your laptop. Start a new tmux session with `tmux new` and we'll name this session `nextflow`. + +``` +tmux new -n nextflow +``` + +Now you can run pipelines with abandon! + +### 2. Make a GitHub repo for your workflows (optional :) + +To make sharing your pipelines and commands easy between your teammates, it's best to share code in a GitHub repository. One way is to store the commands in a Makefile ([example](https://github.com/czbiohub/kh-workflows/blob/master/nf-kmer-similarity/Makefile)) which can contain multiple `nextflow run` commands so that you don't need to remember the S3 bucket or output directory for every single one. [Makefiles](https://kbroman.org/minimal_make/) are broadly used in the software community for running many complex commands. Makefiles can have a lot of dependencies and be confusing, so we're only going to write *simple* Makefiles. + +``` +rnaseq: + nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ + +human_mouse_zebrafish: + nextflow run czbiohub/nf-kmer-similarity -latest -profile aws \ + --samples s3://kmer-hashing/hematopoeisis/smartseq2/human_mouse_zebrafish/samples.csv + + +merkin2012_aws: + nextflow run czbiohub/nf-kmer-similarity -latest --sra "SRP016501" \ + -r olgabot/support-csv-directory-or-sra \ + -profile aws +``` + +In this example, one would run the `rnaseq` rule and the nextflow command beneath it with: + +``` +make rnaseq +``` + +If one wanted to run a different command, e.g. `human_mouse_zebrafish`, they would specify that command instead. For example: + +``` +make human_mouse_zebrafish +``` + +Makefiles are a very useful way of storing longer commands with short mnemonic words. + + +Once you [create a new repository](https://github.com/organizations/czbiohub/repositories/new) (best to initialize with a `.gitignore`, license - MIT and `README`), clone that repository to your EC2 instance. For example, if the repository is called `kh-workflows`, this is what the command would look like: + +``` +git clone https://github.com/czbiohub/kh-workflows +``` + +Now both create and edit a `Makefile`: + +``` +cd +nano Makefile +``` + +Write your rule with a colon after it, and on the next line must be a **tab**, not spaces. Once you're done, exit the program (the `^` command shown in nano means "Control"), write the file, add it to git, commit it, and push it up to GitHub. + + +``` +git add Makefile +git commit -m "Added makefile" +git push origin master +``` + + +### 3. Run your workflow!! + +Remember to specify `-profile czbiohub_aws` to grab the CZ Biohub-specific AWS configurations, and an `--outdir` with an AWS S3 bucket so you don't run out of space on your small AMI + +``` +nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ +``` + +### 4. If you lose connection, how do you restart the jobs? + +If you close your laptop, get onto the train, or lose WiFi connection, you may lose connection to AWS and may need to restart the jobs. To reattach, use the command `tmux attach` and you should see your Nextflow output! To get the named session, use: + +``` +tmux attach -n nextflow +``` + +To restart the jobs from where you left off, add the `-resume` flag to your `nextflow` command: + + +``` +nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ \ + -resume +``` + +It's important that this command be re-run from the same directory as there is a "hidden" `.nextflow` folder that contains all the metadata and information about previous runs. + +## iGenomes specific configuration + +A local copy of the iGenomes resource has been made available on `s3://czbiohub-reference/igenomes` (in `us-west-2` region) so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline. +You can do this by simply using the `--genome ` parameter. + +For Human and Mouse, we use [GENCODE](https://www.gencodegenes.org/) gene annotations. This doesn't change how you would specify the genome name, only that the pipelines run with the `czbiohub_aws` profile would be with GENCODE rather than iGenomes. + + +>NB: You will need an account to use the HPC cluster on PROFILE CLUSTER in order to run the pipeline. If in doubt contact IT. + +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From cfcb533b959d5fb2c4deb5750c6bb5645d5db009 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Tue, 25 Jun 2019 19:02:49 -0700 Subject: [PATCH 17/40] Add czbiohub_aws to list of configs" --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index dfcb4d2..0bfc474 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -16,6 +16,7 @@ profiles { ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } + czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From d1983f8acd15354160e60458355eee2cf75f9b56 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 26 Jun 2019 17:56:26 -0700 Subject: [PATCH 18/40] Use new and improved GENCODE+ERCC annotation --- conf/czbiohub_aws.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index af7707a..a89588b 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -65,13 +65,13 @@ params { genomes { 'GRCh38' { fasta = "${params.gencode_base}/human/v30/GRCh38.p12.genome.ERCC92.fa" - gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gene_type.gtf" + gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gtf" transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" star = "${params.gencode_base}/human/v30/STARIndex/" } 'GRCm38' { fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" - gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gene_type.gtf" + gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gtf" transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" } } From 70b5205d153f4313bdf586837d0eb81e8b5f2607 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Thu, 27 Jun 2019 15:02:47 -0700 Subject: [PATCH 19/40] Add human salmon index and mouse star index --- conf/czbiohub_aws.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index a89588b..bd0228d 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -68,11 +68,13 @@ params { gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gtf" transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" star = "${params.gencode_base}/human/v30/STARIndex/" + salmon_index = "${params.gencode_base}/human/v30/salmon_index/" } 'GRCm38' { fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gtf" transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" + start = "${params.gencode_base}/mouse/vM21/STARIndex/" } } From 6a2d2003f0fed491ccabc130c36e6301616983fa Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 1 Jul 2019 09:25:46 -0700 Subject: [PATCH 20/40] Add high priority AWS batch queue to CZBiohub AWS --- conf/czbiohub_aws.config | 2 +- conf/czbiohub_aws_highpriority.config | 127 ++++++++++++++++++++++++++ nfcore_custom.config | 1 + 3 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 conf/czbiohub_aws_highpriority.config diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index bd0228d..d20b405 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -20,7 +20,7 @@ docker { process { executor = 'awsbatch' - queue = 'nextflow' + queue = 'default-971039e0-830c-11e9-9e0b-02c5b84a8036' errorStrategy = 'ignore' } diff --git a/conf/czbiohub_aws_highpriority.config b/conf/czbiohub_aws_highpriority.config new file mode 100644 index 0000000..2bf9a4d --- /dev/null +++ b/conf/czbiohub_aws_highpriority.config @@ -0,0 +1,127 @@ +/* + * ------------------------------------------------- + * Nextflow config file for Chan Zuckerberg Biohub + * ------------------------------------------------- + * Defines reference genomes, using iGenome paths + * Imported under the default 'standard' Nextflow + * profile in nextflow.config + */ + + //Profile config names for nf-core/configs + params { + config_profile_description = 'Chan Zuckerberg Biohub AWS Batch profile provided by nf-core/configs.' + config_profile_contact = 'Olga Botvinnik (@olgabot)' + config_profile_url = 'https://www.czbiohub.org/' + } + +docker { + enabled = true +} + +process { + executor = 'awsbatch' + queue = 'highpriority-971039e0-830c-11e9-9e0b-02c5b84a8036' + errorStrategy = 'ignore' +} + +workDir = "s3://czb-nextflow/intermediates/" + +aws.region = 'us-west-2' +executor.awscli = '/home/ec2-user/miniconda/bin/aws' +params.tracedir = './' + +params { + saveReference = true + + // Largest SPOT instances available on AWS: https://ec2instances.info/ + max_memory = 1952.GB + max_cpus = 96 + max_time = 240.h + + // Compatible with multiple versions of rnaseq pipeline + seq_center = "czbiohub" + seqCenter = "czbiohub" + + // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + igenomes_base = "s3://czbiohub-reference/igenomes" + + // GENCODE (human + mouse) reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + gencode_base = "s3://czbiohub-reference/gencode" + transgenes_base = "s3://czbiohub-reference/transgenes" + + // AWS configurations + awsregion = "us-west-2" + awsqueue = "nextflow" + + igenomesIgnore = true + + fc_extra_attributes = 'gene_name' + fc_group_features = 'gene_id' + fc_group_features_type = 'gene_type' + + // GENCODE GTF and fasta files + genomes { + 'GRCh38' { + fasta = "${params.gencode_base}/human/v30/GRCh38.p12.genome.ERCC92.fa" + gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gtf" + transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" + star = "${params.gencode_base}/human/v30/STARIndex/" + salmon_index = "${params.gencode_base}/human/v30/salmon_index/" + } + 'GRCm38' { + fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" + gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gtf" + transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" + start = "${params.gencode_base}/mouse/vM21/STARIndex/" + } + } + + transgenes { + 'ChR2' { + fasta = "${params.transgenes_base}/ChR2/ChR2.fa" + gtf = "${params.transgenes_base}/ChR2/ChR2.gtf" + } + 'Cre' { + fasta = "${params.transgenes_base}/Cre/Cre.fa" + gtf = "${params.transgenes_base}/Cre/Cre.gtf" + } + 'ERCC' { + fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa" + gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf" + } + 'GCaMP6m' { + fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa" + gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf" + } + 'GFP' { + fasta = "${params.transgenes_base}/Gfp/Gfp.fa" + gtf = "${params.transgenes_base}/Gfp/Gfp.gtf" + } + 'NpHR' { + fasta = "${params.transgenes_base}/NpHR/NpHR.fa" + gtf = "${params.transgenes_base}/NpHR/NpHR.gtf" + } + 'RCaMP' { + fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa" + gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf" + } + 'RGECO' { + fasta = "${params.transgenes_base}/RGECO/RGECO.fa" + gtf = "${params.transgenes_base}/RGECO/RGECO.gtf" + } + 'Tdtom' { + fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa" + gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf" + } + 'Car-T' { + fasta = "${params.transgenes_base}/car-t/car-t.fa" + gtf = "${params.transgenes_base}/car-t/car-t.gtf" + } + 'zsGreen' { + fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa" + gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf" + } + } +} diff --git a/nfcore_custom.config b/nfcore_custom.config index 0bfc474..de5961c 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -17,6 +17,7 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } + czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From 45d36a150661b4f33a3847d345f17d41b74fa8ee Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 3 Jul 2019 09:21:16 +0200 Subject: [PATCH 21/40] Update cfc.config Add Weblog for CFC cluster --- conf/cfc.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/cfc.config b/conf/cfc.config index dd69647..2f9047d 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -14,6 +14,11 @@ process { executor = 'slurm' } +weblog{ + enabled = true + url = 'http://services.qbic.uni-tuebingen.de:8080/workflows' +} + params { igenomes_base = '/nfsmounts/igenomes' max_memory = 60.GB From 917ec0863d5e427940ec8a19ecdc258df786aef1 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 3 Jul 2019 16:01:03 -0700 Subject: [PATCH 22/40] add default trimming --- conf/czbiohub_aws.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index d20b405..71d1a9b 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -61,6 +61,8 @@ params { fc_group_features = 'gene_id' fc_group_features_type = 'gene_type' + trim_pattern = '_+S\\d+' + // GENCODE GTF and fasta files genomes { 'GRCh38' { From 069963ff8a5ce9f6cad3a7be71e44782f3417ee4 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 3 Jul 2019 16:01:47 -0700 Subject: [PATCH 23/40] Make czbiohub_aws_highpriority only set the AWS Batch queue and nothing else --- conf/czbiohub_aws_highpriority.config | 115 -------------------------- nfcore_custom.config | 5 +- 2 files changed, 4 insertions(+), 116 deletions(-) diff --git a/conf/czbiohub_aws_highpriority.config b/conf/czbiohub_aws_highpriority.config index 2bf9a4d..5ab796a 100644 --- a/conf/czbiohub_aws_highpriority.config +++ b/conf/czbiohub_aws_highpriority.config @@ -7,121 +7,6 @@ * profile in nextflow.config */ - //Profile config names for nf-core/configs - params { - config_profile_description = 'Chan Zuckerberg Biohub AWS Batch profile provided by nf-core/configs.' - config_profile_contact = 'Olga Botvinnik (@olgabot)' - config_profile_url = 'https://www.czbiohub.org/' - } - -docker { - enabled = true -} - process { - executor = 'awsbatch' queue = 'highpriority-971039e0-830c-11e9-9e0b-02c5b84a8036' - errorStrategy = 'ignore' -} - -workDir = "s3://czb-nextflow/intermediates/" - -aws.region = 'us-west-2' -executor.awscli = '/home/ec2-user/miniconda/bin/aws' -params.tracedir = './' - -params { - saveReference = true - - // Largest SPOT instances available on AWS: https://ec2instances.info/ - max_memory = 1952.GB - max_cpus = 96 - max_time = 240.h - - // Compatible with multiple versions of rnaseq pipeline - seq_center = "czbiohub" - seqCenter = "czbiohub" - - // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket - // No final slash because it's added later - igenomes_base = "s3://czbiohub-reference/igenomes" - - // GENCODE (human + mouse) reference file paths on CZ Biohub reference s3 bucket - // No final slash because it's added later - gencode_base = "s3://czbiohub-reference/gencode" - transgenes_base = "s3://czbiohub-reference/transgenes" - - // AWS configurations - awsregion = "us-west-2" - awsqueue = "nextflow" - - igenomesIgnore = true - - fc_extra_attributes = 'gene_name' - fc_group_features = 'gene_id' - fc_group_features_type = 'gene_type' - - // GENCODE GTF and fasta files - genomes { - 'GRCh38' { - fasta = "${params.gencode_base}/human/v30/GRCh38.p12.genome.ERCC92.fa" - gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gtf" - transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" - star = "${params.gencode_base}/human/v30/STARIndex/" - salmon_index = "${params.gencode_base}/human/v30/salmon_index/" - } - 'GRCm38' { - fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" - gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gtf" - transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" - start = "${params.gencode_base}/mouse/vM21/STARIndex/" - } - } - - transgenes { - 'ChR2' { - fasta = "${params.transgenes_base}/ChR2/ChR2.fa" - gtf = "${params.transgenes_base}/ChR2/ChR2.gtf" - } - 'Cre' { - fasta = "${params.transgenes_base}/Cre/Cre.fa" - gtf = "${params.transgenes_base}/Cre/Cre.gtf" - } - 'ERCC' { - fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa" - gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf" - } - 'GCaMP6m' { - fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa" - gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf" - } - 'GFP' { - fasta = "${params.transgenes_base}/Gfp/Gfp.fa" - gtf = "${params.transgenes_base}/Gfp/Gfp.gtf" - } - 'NpHR' { - fasta = "${params.transgenes_base}/NpHR/NpHR.fa" - gtf = "${params.transgenes_base}/NpHR/NpHR.gtf" - } - 'RCaMP' { - fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa" - gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf" - } - 'RGECO' { - fasta = "${params.transgenes_base}/RGECO/RGECO.fa" - gtf = "${params.transgenes_base}/RGECO/RGECO.gtf" - } - 'Tdtom' { - fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa" - gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf" - } - 'Car-T' { - fasta = "${params.transgenes_base}/car-t/car-t.fa" - gtf = "${params.transgenes_base}/car-t/car-t.gtf" - } - 'zsGreen' { - fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa" - gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf" - } - } } diff --git a/nfcore_custom.config b/nfcore_custom.config index de5961c..72dca30 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -17,7 +17,10 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } - czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } + czbiohub_aws_highpriority { + includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" + includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" + } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From cb507177d5ceca20722bea2acda201c040d9300b Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Fri, 5 Jul 2019 08:03:37 +0200 Subject: [PATCH 24/40] Updating CCGA config file --- conf/ccga.config | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/conf/ccga.config b/conf/ccga.config index f29db9c..b9130e0 100644 --- a/conf/ccga.config +++ b/conf/ccga.config @@ -2,15 +2,20 @@ params { config_profile_description = 'CCGA cluster profile provided by nf-core/configs.' config_profile_contact = 'Marc Hoeppner (@marchoeppner)' - config_profile_url = 'https://www.ikmb.uni-kiel.de/' + config_profile_url = 'https://www.ccga.uni-kiel.de/' } /* * ------------------------------------------------- - * Nextflow config file with environment modules for RZCluster in Kiel + * Nextflow config file for CCGA cluster in Kiel * ------------------------------------------------- */ +singularity { + enabled = true + runOptions = "-B /ifs -B /scratch -B /work_beegfs" +} + executor { queueSize=100 } @@ -29,4 +34,7 @@ params { // illumina iGenomes reference file paths on RZCluster igenomes_base = '/ifs/data/nfs_share/ikmb_repository/references/iGenomes/references/' saveReference = true + max_memory = 128.GB + max_cpus = 16 + max_time = 120.h } From d66bf4e4786799301846b34dfd9602d17c4ce3ce Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Fri, 5 Jul 2019 08:28:18 +0200 Subject: [PATCH 25/40] Updating CCGA documentation --- docs/ccga.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 docs/ccga.md diff --git a/docs/ccga.md b/docs/ccga.md new file mode 100644 index 0000000..4a38bd9 --- /dev/null +++ b/docs/ccga.md @@ -0,0 +1,18 @@ +# nf-core/configs: CCGA Configuration + +Deployment and testing of nf-core pipelines at the CCGA cluster is on-going. + +To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the BINAC cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on the cluster. You can do this by issuing the commands below: + +```bash +## Load Nextflow and Singularity environment modules +module purge +module load IKMB +module load Java/1.8.0 +module load Nextflow +module load singularity3.1.0 +``` + +>NB: Access to the CCGA cluster is restricted to IKMB/CCGA employes. Please talk to Marc Hoeppner to get access (@marchoeppner). From fe0153befd41e7cd054a398d64dac7da89f1f45d Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Fri, 5 Jul 2019 08:30:02 +0200 Subject: [PATCH 26/40] Updating CCGA documentation --- docs/ccga.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ccga.md b/docs/ccga.md index 4a38bd9..1157cb3 100644 --- a/docs/ccga.md +++ b/docs/ccga.md @@ -2,7 +2,7 @@ Deployment and testing of nf-core pipelines at the CCGA cluster is on-going. -To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the BINAC cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on the cluster. You can do this by issuing the commands below: From 9106559b2f319c0f78a021bc28abd714ab957c40 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 15 Jul 2019 17:53:11 +0200 Subject: [PATCH 27/40] Update cfc.config --- conf/cfc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index 2f9047d..6fc3b26 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -16,7 +16,7 @@ process { weblog{ enabled = true - url = 'http://services.qbic.uni-tuebingen.de:8080/workflows' + url = 'http://services.qbic.uni-tuebingen.de:8080/workflowservice/workflows' } params { From 1b30c5a4d24301ad1100e305f4ad1885b9f22b3f Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Wed, 24 Jul 2019 11:32:02 +0200 Subject: [PATCH 28/40] Add the Pasteur profile --- conf/pasteur.config | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 conf/pasteur.config diff --git a/conf/pasteur.config b/conf/pasteur.config new file mode 100644 index 0000000..03971dc --- /dev/null +++ b/conf/pasteur.config @@ -0,0 +1,24 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'The Institut Pasteur HPC cluster profile' + config_profile_contact = 'Remi Planel (@rplanel)' + config_profile_url = 'https://research.pasteur.fr/en/service/tars-cluster/' +} + +singularity { + enabled = true + autoMounts = true + runOptions = '-B /local/scratch:/tmp' +} + +process { + executor = 'slurm' + clusterOptions = { "-A $params.project ${params.clusterOptions ?: ''}" } +} + +params { + igenomesIgnore = true + max_memory = 256.GB + max_cpus = 28 + max_time = 24.h +} From 1cb132a4572dc85bd7478bdb881cb1934e0cdf0f Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Wed, 24 Jul 2019 11:32:25 +0200 Subject: [PATCH 29/40] Define a profile for the Pasteur config --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 41b0ade..a9c0729 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -33,6 +33,7 @@ profiles { uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } prince { includeConfig "${params.custom_config_base}/conf/prince.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } + pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } } // If user hostnames contain one of these substring and they are From fbb33fde63700e1f76852ad7df2780bba4b7ec6a Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Wed, 24 Jul 2019 11:43:08 +0200 Subject: [PATCH 30/40] Details --- conf/pasteur.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pasteur.config b/conf/pasteur.config index 03971dc..2548d3f 100644 --- a/conf/pasteur.config +++ b/conf/pasteur.config @@ -2,7 +2,7 @@ params { config_profile_description = 'The Institut Pasteur HPC cluster profile' config_profile_contact = 'Remi Planel (@rplanel)' - config_profile_url = 'https://research.pasteur.fr/en/service/tars-cluster/' + config_profile_url = 'https://research.pasteur.fr/en/service/tars-cluster' } singularity { From 71bc98afaef852562d5664d7861cc31cb870db79 Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Mon, 29 Jul 2019 12:06:29 +0200 Subject: [PATCH 31/40] Adding config file for CCGA dx cluster --- docs/ccga_dx.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/ccga_dx.md diff --git a/docs/ccga_dx.md b/docs/ccga_dx.md new file mode 100644 index 0000000..62bf29c --- /dev/null +++ b/docs/ccga_dx.md @@ -0,0 +1,9 @@ +# nf-core/configs: CCGA DX Configuration + +Deployment and testing of nf-core pipelines at the CCGA DX cluster is on-going. + +To use, run the pipeline with `-profile ccga_dx`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to have Nextflow installed. + +>NB: Access to the CCGA DX cluster is restricted to IKMB/CCGA employes. Please talk to Marc Hoeppner to get access (@marchoeppner). From b31db18dfb0b790aecc81c3685a5cc30a57fccec Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Mon, 29 Jul 2019 12:08:07 +0200 Subject: [PATCH 32/40] Adding config file for CCGA dx cluster --- README.md | 1 + conf/ccga_dx.config | 37 +++++++++++++++++++++++++++++++++++++ nfcore_custom.config | 1 + 3 files changed, 39 insertions(+) create mode 100644 conf/ccga_dx.config diff --git a/README.md b/README.md index 73b01be..d865c89 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ Currently documentation is available for the following clusters: * [BINAC](docs/binac.md) * [BIGPURPLE](docs/bigpurple.md) * [CCGA](docs/ccga.md) +* [CCGA-DX]/docs/ccga_dx.md) * [CFC](docs/binac.md) * [CRICK](docs/crick.md) * [GIS](docs/gis.md) diff --git a/conf/ccga_dx.config b/conf/ccga_dx.config new file mode 100644 index 0000000..0af95e7 --- /dev/null +++ b/conf/ccga_dx.config @@ -0,0 +1,37 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'CCGA DX cluster profile provided by nf-core/configs.' + config_profile_contact = 'Marc Hoeppner (@marchoeppner)' + config_profile_url = 'https://www.ccga.uni-kiel.de/' +} + +/* + * ------------------------------------------------- + * Nextflow config file for CCGA cluster in Kiel + * ------------------------------------------------- + */ + +singularity { + enabled = true +} + +executor { + queueSize=100 +} + +process { + + // Global process config + executor = 'slurm' + queue = 'htc' + +} + +params { + // illumina iGenomes reference file paths on DX Cluster + igenomes_base = '/mnt/ld_ng_out/sukmb352/references/iGenomes/references/' + saveReference = true + max_memory = 250.GB + max_cpus = 20 + max_time = 240.h +} diff --git a/nfcore_custom.config b/nfcore_custom.config index 41b0ade..d4ae35b 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -14,6 +14,7 @@ params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/$ profiles { binac { includeConfig "${params.custom_config_base}/conf/binac.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } + ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } From 7d5de651aa238c77f6bee3347622d34c1aa8a942 Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Mon, 29 Jul 2019 12:45:08 +0200 Subject: [PATCH 33/40] Update README.md Co-Authored-By: Harshil Patel --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d865c89..d7b43fb 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Currently documentation is available for the following clusters: * [BINAC](docs/binac.md) * [BIGPURPLE](docs/bigpurple.md) * [CCGA](docs/ccga.md) -* [CCGA-DX]/docs/ccga_dx.md) +* [CCGA_DX](/docs/ccga_dx.md) * [CFC](docs/binac.md) * [CRICK](docs/crick.md) * [GIS](docs/gis.md) From b74f020f14d3254d67d85edaa2a3c8d20dfea389 Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Mon, 29 Jul 2019 12:45:30 +0200 Subject: [PATCH 34/40] Update docs/ccga_dx.md Co-Authored-By: Harshil Patel --- docs/ccga_dx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ccga_dx.md b/docs/ccga_dx.md index 62bf29c..0cea81e 100644 --- a/docs/ccga_dx.md +++ b/docs/ccga_dx.md @@ -2,7 +2,7 @@ Deployment and testing of nf-core pipelines at the CCGA DX cluster is on-going. -To use, run the pipeline with `-profile ccga_dx`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile ccga_dx`. This will download and launch the [`ccga_dx.config`](../conf/ccga_dx.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Before running the pipeline you will need to have Nextflow installed. From 96ea16ddbd9082f630c4836d2cb032542c15151c Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 15:14:18 +0200 Subject: [PATCH 35/40] Ignore vscode workspace --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 07c0144..8aa0735 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ work/ data/ results/ .DS_Store +*.code-workspace \ No newline at end of file From b4cffe0455d1523ad179fd0d125857511e722216 Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 15:15:07 +0200 Subject: [PATCH 36/40] Remove buggy cluster option --- conf/pasteur.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/pasteur.config b/conf/pasteur.config index 2548d3f..1d95131 100644 --- a/conf/pasteur.config +++ b/conf/pasteur.config @@ -13,7 +13,6 @@ singularity { process { executor = 'slurm' - clusterOptions = { "-A $params.project ${params.clusterOptions ?: ''}" } } params { From 6f264ac2caca9429e27e72c1347d96dfdfcd9457 Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 15:15:30 +0200 Subject: [PATCH 37/40] Add pasteur profile doc --- docs/pasteur.md | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/pasteur.md diff --git a/docs/pasteur.md b/docs/pasteur.md new file mode 100644 index 0000000..153f4d2 --- /dev/null +++ b/docs/pasteur.md @@ -0,0 +1,54 @@ +# nf-core/configs: Institut Pasteur Configuration + +All nf-core pipelines have been successfully configured for use on the tars cluster at the Institut Pasteur. + +To use, run the pipeline with `-profile pasteur`. This will download and launch the [`pasteur.config`](../conf/pasteur.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + + + +## Running the workflow on the Pasteur cluster + +>Nextflow is not installed by default on the Pasteur cluster. +- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) + + +>Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. +Nextflow shouldn't run directly on the submission node but on a compute node. +The compute nodes don't have access to internet so you need to run it offline. To do that: +1. Create a virtualenv to install nf-core in order to get the workflow +```bash +module purge +module load Python/3.6.0 +module load java +module load singularity +cd /path/to/nf-core/workflows +virtualenv .venv -p python3 +. .venv/bin/activate +``` +2. Install nf-core: [here](https://nf-co.re/tools#installation) +3. Get nf-core pipeline and container: [here](https://nf-co.re/tools#downloading-pipelines-for-offline-use) +4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/rnaseq/blob/master/docs/usage.md#--custom_config_base) +5. Run nextflow on a compute node: +```bash +# create a terminal +tmux + +# Get a compute node +salloc + +# Load the dependencies if not done before +module purge +module load java +module load singularity + +# Run nextflow workflow +nextflow run \\ +/path/to/pipeline-dir/from/step/2/workflow \\ +-resume +-profile pasteur \\ +-with-singularity /path/to/pipeline-dir/from/step/3/singularity-images/singularity.img \\ +--email my-email@pasteur.fr \\ +--custom_config_base /path/to/configs/from/step/4/ \\ +-c my-specific.config +... +``` From 64e78a3dc15ef11224b8da0bf7ed64b9c30e816b Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 15:17:25 +0200 Subject: [PATCH 38/40] Update the doc --- docs/pasteur.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pasteur.md b/docs/pasteur.md index 153f4d2..676199d 100644 --- a/docs/pasteur.md +++ b/docs/pasteur.md @@ -43,7 +43,7 @@ module load singularity # Run nextflow workflow nextflow run \\ -/path/to/pipeline-dir/from/step/2/workflow \\ +/path/to/pipeline-dir/from/step/3/workflow \\ -resume -profile pasteur \\ -with-singularity /path/to/pipeline-dir/from/step/3/singularity-images/singularity.img \\ From f9a2baf6143fec9325cd8fea10eb4dcdd478470e Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 16:10:33 +0200 Subject: [PATCH 39/40] Add link to Pasteur doc --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 73b01be..edab0b5 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ Currently documentation is available for the following clusters: * [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UPPMAX](docs/uppmax.md) * [UZH](docs/uzh.md) +* [PASTEUR](docs/pasteur.md) ### Uploading to `nf-core/configs` From be28a34839f795ec9cff95102bc1b26d1ca231ae Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 16:12:30 +0200 Subject: [PATCH 40/40] Updat Pasteur doc --- docs/pasteur.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/pasteur.md b/docs/pasteur.md index 676199d..554761b 100644 --- a/docs/pasteur.md +++ b/docs/pasteur.md @@ -8,14 +8,16 @@ To use, run the pipeline with `-profile pasteur`. This will download and launch ## Running the workflow on the Pasteur cluster ->Nextflow is not installed by default on the Pasteur cluster. +Nextflow is not installed by default on the Pasteur cluster. - Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) ->Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. +Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. Nextflow shouldn't run directly on the submission node but on a compute node. -The compute nodes don't have access to internet so you need to run it offline. To do that: -1. Create a virtualenv to install nf-core in order to get the workflow +The compute nodes don't have access to internet so you need to run it offline. + +To do that: +1. Create a virtualenv to install nf-core ```bash module purge module load Python/3.6.0