From 6c5a5c8bcb5b548c00cb25a5ee9016853a809603 Mon Sep 17 00:00:00 2001 From: Marc Hoeppner Date: Mon, 24 Feb 2020 12:01:43 +0100 Subject: [PATCH 01/87] Removing old CCGA profile from docs and linting Removing CCGA profile from master config Removing CCGA profile from master config --- .github/workflows/main.yml | 2 +- README.md | 1 - conf/ccga.config | 41 -------------------------------------- docs/ccga.md | 18 ----------------- nfcore_custom.config | 1 - 5 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 conf/ccga.config delete mode 100644 docs/ccga.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5fb2601..6f96a98 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/README.md b/README.md index 3011a00..7634409 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,6 @@ Currently documentation is available for the following systems: * [BIGPURPLE](docs/bigpurple.md) * [BINAC](docs/binac.md) * [CBE](docs/cbe.md) -* [CCGA](docs/ccga.md) * [CCGA_DX](docs/ccga_dx.md) * [CCGA_MED](docs/ccga_med.md) * [CFC](docs/binac.md) diff --git a/conf/ccga.config b/conf/ccga.config deleted file mode 100644 index 6163626..0000000 --- a/conf/ccga.config +++ /dev/null @@ -1,41 +0,0 @@ -//Profile config names for nf-core/configs -params { - config_profile_description = 'CCGA cluster profile provided by nf-core/configs.' - config_profile_contact = 'Marc Hoeppner (@marchoeppner)' - config_profile_url = 'https://www.ccga.uni-kiel.de/' -} - -/* - * ------------------------------------------------- - * Nextflow config file for CCGA cluster in Kiel - * ------------------------------------------------- - */ - -singularity { - enabled = true - runOptions = "-B /ifs -B /scratch -B /work_beegfs" - cacheDir = "/ifs/data/nfs_share/ikmb_repository/singularity_cache/" -} - -executor { - queueSize=100 -} - -process { - - // Global process config - executor = 'slurm' - queue = 'ikmb_a' - - clusterOptions = { "--qos=ikmb_a" } - -} - -params { - // illumina iGenomes reference file paths on RZCluster - igenomes_base = '/ifs/data/nfs_share/ikmb_repository/references/iGenomes/references/' - saveReference = true - max_memory = 128.GB - max_cpus = 16 - max_time = 120.h -} diff --git a/docs/ccga.md b/docs/ccga.md deleted file mode 100644 index 798df29..0000000 --- a/docs/ccga.md +++ /dev/null @@ -1,18 +0,0 @@ -# nf-core/configs: CCGA Configuration - -Deployment and testing of nf-core pipelines at the CCGA cluster is on-going. - -To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. - -Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on the cluster. You can do this by issuing the commands below: - -```bash -## Load Nextflow and Singularity environment modules -module purge -module load IKMB -module load Java/1.8.0 -module load Nextflow -module load singularity3.1.0 -``` - ->NB: Access to the CCGA cluster is restricted to IKMB/CCGA employes. Please talk to Marc Hoeppner to get access (@marchoeppner). diff --git a/nfcore_custom.config b/nfcore_custom.config index 8b4a3a9..0fcbe66 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -17,7 +17,6 @@ profiles { bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } cbe { includeConfig "${params.custom_config_base}/conf/cbe.config" } - ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } ccga_med { includeConfig "${params.custom_config_base}/conf/ccga_med.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } From e2e4dd05d318c6a537f5dd17f6a1ec82c5a32167 Mon Sep 17 00:00:00 2001 From: phue Date: Thu, 5 Mar 2020 16:34:20 +0100 Subject: [PATCH 02/87] cbe: move container cache to different filesystem this is a workaround for squashfs errors that occur when containers are run from beeGFS --- conf/cbe.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cbe.config b/conf/cbe.config index ea24e5b..5888202 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -14,7 +14,7 @@ process { singularity { enabled = true - cacheDir = '/scratch-cbe/shared/containers' + cacheDir = '/resources/containers' } params { From 8a9d3ad8a0ca66ef35276adacc1facac4d1b0129 Mon Sep 17 00:00:00 2001 From: Evan Floden Date: Fri, 6 Mar 2020 05:47:29 +0000 Subject: [PATCH 03/87] First commit of Google Cloud config --- .github/workflows/main.yml | 2 +- README.md | 1 + conf/google.config | 21 +++++++++++++++++++++ docs/google.md | 30 ++++++++++++++++++++++++++++++ docs/images/google-cloud-logo.png | Bin 0 -> 13226 bytes nfcore_custom.config | 1 + 6 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 conf/google.config create mode 100644 docs/google.md create mode 100644 docs/images/google-cloud-logo.png diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5fb2601..d68f44f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'google', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/README.md b/README.md index 0e58b09..b49edb7 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,7 @@ Currently documentation is available for the following systems: * [GENOTOUL](docs/genotoul.md) * [GENOUEST](docs/genouest.md) * [GIS](docs/gis.md) +* [GOOGLE](docs/google.md) * [HEBBE](docs/hebbe.md) * [KRAKEN](docs/kraken.md) * [MUNIN](docs/munin.md) diff --git a/conf/google.config b/conf/google.config new file mode 100644 index 0000000..4333d86 --- /dev/null +++ b/conf/google.config @@ -0,0 +1,21 @@ +// Nextflow config file for running on Google Cloud Life Sciences +params { + config_profile_description = 'Google Cloud Life Sciences Profile' + config_profile_contact = 'Evan Floden, Seqera Labs (@evanfloden)' + config_profile_url = 'https://cloud.google.com/life-sciences' + + google_zone = 'europe-west2-c' + google_bucket = false + google_debug = false + google_preemptible = true +} + +process.executor = 'google-lifesciences' +google.zone = params.google_zone +google.lifeSciences.debug = params.google_debug +workDir = params.google_bucket +google.lifeSciences.preemptible = params.google_preemptible +if (google.lifeSciences.preemptible) { + process.errorStrategy = { task.exitStatus==14 ? 'retry' : 'terminate' } + process.maxRetries = 5 +} \ No newline at end of file diff --git a/docs/google.md b/docs/google.md new file mode 100644 index 0000000..d2b0f51 --- /dev/null +++ b/docs/google.md @@ -0,0 +1,30 @@ +# nf-core/configs: Google Cloud Life Sciences Configuration +![Google Cloud](images/google-cloud-logo.png "https://cloud.google.com/life-sciences/docs/tutorials/nextflow") + +To be used with the `google` profile by specifying the `-profile google` when running nf-core pipelines. + +## Quick Start +``` +GOOGLE_APPLICATION_CREDENTIALS=.json +NXF_MODE=google +nextflow run nf-core/rnaseq -profile google,test --google_bucket +``` + +## Required Parameters +#### `--google-bucket` +The Google Cloud Storage bucket location to be used as a Nextflow work directory. Can also be specified with (`-w gs://your_bucket/work`). + +## Optional Parameters +#### `--google-zone` +The Google zone where the computation is executed in Compute Engine VMs. Multiple zones can be provided separating them by a comma. Default (`europe-west2-c`). + +#### `--google-preemptible` +Enables the usage of preemptible virtual machines with a retry error statergy for up to 5 retries. Default (`true`) + +#### `--google-debug` +Copies the /google debug directory from the VM to the task bucket directory. Useful for debugging. Default (`false`). + +## Google Cloud Life Sciences Setup +Please refer to the [Google Cloud](https://cloud.google.com/life-sciences/docs/tutorials/nextflow) and [Nextflow](https://www.nextflow.io/docs/latest/google.html#cloud-life-sciences) documentation which describe how to setup the Google Cloud environment. + + diff --git a/docs/images/google-cloud-logo.png b/docs/images/google-cloud-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..07c9a325b254d93f3fe4e94abb5cbdd9162afd92 GIT binary patch literal 13226 zcmZ{LWmFtZ(C#em?hxEvg9ZyO!QI^@xVuYm4=%yoC1}tM9^5SicY=SL_ulWE`{$lL zXJ=-&d#0zlx}K`1s-u-ZNuweWAprn@Dk~$Q3IL!R$oVG(ILP;Q0N3;+U|0D$lOwNph1@&}x$ytD-H z_U}{B^DPAcIQwNK#MHf3P7S>M)HLqi@}81wmEYs5zsGzhZJ`2#uReZsc4)m;peQ4k zt$ExYPjv_$LiN<2e$S$qjdNeQ`-MDb_x&r*y}Q1su{kCNJ&Qw%=hS(EX4w048bw2` zjeD-YS?}w3@AEv*U-DefI(}maP%I%g?61#Amtzt^570xQ!a1`CiIJOg3y0+a>Oc<2 z9_~_#-Ct>IihAxhwVjBd@sA8alYuZqY+8jA=z_`Fq+2V}u3l9q{>Z4;p)Ew9Fl+pts3YF0?dq9pjU|kyJuiE?tw!fxd z<4?a^T7^t@lL6`*>IRdeChZ`AD4!w>skt1#KanwC-N zgjUBRs;j6=bl^qM>!d4l?3s78R^UKbnn5-Ag?%v6-c>g=74>ea~W)aCl zvE(Fkl6wu&QEbbBW06eYivJ^!PArL>q=&HG5zO5lR*3_^8Q@}B=Kg%L`M(YByu~-U zZK~geq=WbhkjEt+DyO0Zzpb^N%I0LKff-Ey6Kf|Fif4z@SE2VAnM^CQk52M_zkG4d zh}`LTqqt3nkKGBg0t3!3fV{kJF#9offfA!yC*J!|LzcW8P=Fq7h_i7%x~>AE*CT+ z3Ek#|&-TdVq#KNx&uaW@jHrdxVqUNP$1b>;MicC0=A*k8Is zuuCNptn(2W0+Hb4d$~%@2PELJ9;dJq-q}-3L;`omG2(wa?-ABMT-Ek4U(<(l2K-U{ zaJGq08~)Q$ehhrXYXHTYcK?*SWitk!FN|(UqkW~<%JwatD3}-yFV)W0 zJix0>K5r|h)mH-pq*=FDm~bbjsUp)z1Mca~7vWr%c%P~=jh<^eNE9=dkoVm1NQhMa z(yKm*h8#|WkxF#Sb8=vMU`>!|Z=Z`>Scqiqa3pR{3sI<|Rme%vjfJMwxeVa*?>~^*?%s)$tMC+Tub7o&?pAQI`76on5VATs@JpXOu+y6QW6*1h!s;8-GP}Zbdqi z#R4!1;wgl{K=9085h(SL^?e=mVI*Mcm!^|pcD%;{whw{bST4jA&?Q~rTEFamuH^QP0~xKlIfahLhN2e1ly28=O- zx&jG#YriSfh>%HWVX?no+4NsH_geE*s2cOT&(3j^cHf`P@nJnj!=|!|c!H6+)Y)DcNtGL|zy{vWL0wT!e06CSk7=3vtKB-I7{^eJ^-b?Y)}=_ny?VeXr-; zYN*DbAXs9$ge(Q{Jv8?KIj2+h^-a(zA*q2oM}j7v9 zmreoz$Cy5!()F}J-_tKO=;|686=ls`B3P6+MorSUXwXe4r--|Xc+RBld!c0UwpV>L zWX#(qCo?&zD|U!}Pv2gKb`9hHvxJU*98M+F5XqfCzMBdL+jIJeRl0`iaoX- zHjSh~0zaO<)gr$UI~_tiy`tX%vXOU{L)lhhY5pd;$$>F1($)DMzT>SxE zYfm3m?oIJ~$*PvYRB#kDNzWmY4o&Ju+X9kyc zc7PvV_vbZ(2?T8~Q83AHrG2EqZMr4Pc{bz2=9!&MyAsHN_H$N@`l1vSF;2 zcKmom)+R=%4BdD*#C5wdSKP@0+xH7oyAG*07+v()R-#2HJ-}zZvX$h*cX771V&mwV zO4AQm=UQ)fUI|iUG|P~(P%b4nk1BGJ~LbMY8@8z#uHh5qP%n}&fcGDEwpdA|VP z*Vi#*k5EQ9J3Fkv#$a2rOTYHJLObuU*)7;D7?btROIcIXGWp+NzRfMC;QHXMEOWPU z?oJg3onMViOr+4RCid?As!xl+HdARzSYS|Cwk)JhnFaw#2&|Y{CgXfos6P9Fp{ba9 z*fgJWUoZFp7BsdNRvT^0cgvd4MtApl69A)P>pNP3UZ(Ct1m;C@DIY^q91S?R^vnr} zN3dHEGcOI0)HWPxGAPh#!TG&$UJ2&YGd%G8pqzr;zAQ7U(aapqxy4ox-;4)e2M7op z$JE)XMLNN*bY2jcVafX87@GP=WsLZ?RJzxPmEw!KC%=#g|C!9@$bF#cT0b)cYU4P* zScQT%{aj@sKu6+YsNm{Dp! zG4&l-;jwAsLLoR#5#6C{_bHig5CPwpSrz@kKRnIC!VdgT-@I&Awq$Z|p`lp+^aYU8 z^qq%gXwwp}*E6PaQU|8zjrD~@d(Pu{4tM=Q8J~Ijir~t`prAu)-5t=)~q`_xS(DURutWlD%7@7T3UkRpLFUxEoVyJJR`$C(ZCC*(lCPO9EHq2=A@)w zPLYEe^@)T^Bm#$hS}R4OaX&59W_@ozL{ck`te{%SQ21cHX?~2&ZtrTSOgD~L|Fb+4 zhL4C4{`}o@MO>-y&X#!YuIv?w?^-Fy>%V-vl3 z9JGlMsHdu05k3NqZPr?)xg@AWx-tN>7B zuaaOGw9e(j_TBJM(n8oQ4-f%VqZW@rka7Wg(7P5aNFOagB;+qkMGQU~vEP^!sH=SebKZ2=Q0B7I~_`6A_H%o zF%icm)bMos@lR|ta%0+{EHthDC-UzNJh6J4X4`~Nu;`r5r1Z5h(S*kO3e>EvT|ycq zF;yY`JE$;^@O>&wx8Z{opk|fu+O;gM1acd1ge|U}bXsHTx9sB^FmB`7nKAp4 zlx+N=YCO6j;?KaF3IE_xS4*5YeDO)mel=pPqyz3_Xk+e7Ddc{wI^C{qc*tbEs#8qp z`L<}#yi$zQ-oFc?!g@FI#URN{^x5QkGZsz=aqDVGKYAOB2FGS-)5 zjI(0!kdMJ$rNdcJek|8KZz-B2BZrW^$SeTM87>j567R6ec5MepZAO20oIU-O;nrS) z;G^|6?`$xhvu**M^}dFFV*3gP2lQ@UaYEUCPd zUOKYAf{Ul5=5BA@hIPhULh-OV3#ZX*yZn7;H|mJ}Oh9(FD0XD9iar@;KK~Wcr&Bb} z&wL~{Xc;4msdt@ezS!}m)8dg6*v-qevhV=q3Mqb}bh>UW=u#(FPw2U*kw@QJ-Hz#+ zA#RvEX8107ROGVNA<{iH74o!c8q5K$1QB5;>3ePVhe*BI0VcwF;-wRt>Wh(%$&j!R zj?>y_S;z%|vX*M?)5EAKkP90qBlF2SPf!#mhAJbldfA_z=|(uH`c7Txbn;mm;f6j) z9eRaWvGVWqA22WX=8uHq_oQ>??(Z{}TUXxYY>M(mtnU-j4g7@(i$!6V`eJ_YPhi!x z%qbJdNRRc&tzUzmVQ>#=OG)qIstxKo*3VEZC1`GX^v|B99t#jC+dLdEr>oKl#bib-B-$KM^lOt_*z zPhe7g{uFXu!;9H_Fgnr8t!?<|w>I>LkuSer30lSm`ZGxxR&P#@7JFf1_g>RNB0rZU z@lt~DsRAB4bo;=T<-Jxko}wpi-c@GrQsS+s;xc!!^#OAt`A$wYI?VGQf91R-2fkMbC=uwNxAeTEIfN^)I9eg( zLV}kY{o(NOpv%OfjP9iNxq>3_Pp=uaReuHeS)DlP!Rb&Q9BT=tCm~kqOBl>DGH1_f ztAiiTG8$?~rsB!x6otmnQmur>9nElYaTzAS9GglUq>rJHf_htYV7cxNnK4%XL4M6+ z9FPU8i4YYZ--a=KsZFK6#wlJuDbAT<~aD8&5$ddY+CDKS^7>BY-&2vO+2 zVP4VzrZj3tY~g+atH&Q4!WXN&iz)ww544>kXoQl0;c!6)>|fushAeWQls;wl{PjH> zIh%9ObLaL*u9=?CLXRJU9!f?tYy)0yM z?!oEQINgEj&mY;Frs_)?gl|l#B6q(0%p1DsNFQ>rvl-t4;D$ZZ)JZ+qLkFP)II9ME zpMe7qpXRrkPze}UkhKq`MCLjC;TFNl7(9kL#!W_b+=fv%>ei>OIkV6{{rF0q-t7`G zy9Fayi7A-_kFJ_q#s`r0Z3Y1z=5-4Mz>I8!RNt>_@F|E2ONe$2FV$0$Mi-JxM%j48 zNcz|Fo4|zrAO^Zowk*0PxX|K$ixKVT2NQOD^8Asvc!d=rdT#xNgQEVsV6B2ILmOy1 z2KSM-Vkf!mz981vdn~Zq@VugG!Lgfo%bGU~hU-~UQc~LeuIu<8sbJ}yFg%hIeK>WA z(|`5!q6gB?%E@?x3(kvk|4I=wEchG9wJH~b?e6b0(l(6?X9;DI`%8|;mqh5u&f++) zykmRR(MszIC`UV)mt99+@B(je#@5N0W9_x;_(TIO9L750ppyxO{yWf##?>hi&4l`4%xQZ)(u7r`&fR zN6BIa@+*HLihlL>789GlJ`qT>gmQC3aliX++~-+_mIV!vUgHNE{khNm*!S`DcEdFn z)(VPG@pqB{gYzeLWf!MvrcV^T=kHX9_yv5?F{6D^mR?kg&p{+&yn}99+w5@7P^p^} z2B!23brVF6oP*xLeN*q=H)>H>S(QKTTSNa%hIozhpgP4EuSe8EP&Hs>a z3q2Ds?RGird#zp2#t-hJ64ApDepjF&kYJ&6q-VI>yljsYY|4;K^Y3jj1M4y2U%pqF zF$%+p1-%tj{RVO2f$v#j0x2ac`#bv!==KHHzfpJH6iu1TltOjx1P>hjQt1(iQ^btX z|5{G-!CdrB55yeDC@r?QV&9rN$(TaBFL2~QY?+7q)#zD_@>CN}4iD=vI5P~bXabcG zagHUuZ9q+}EJ7su)_P^=(WxedcoOy+H7ZZOXS}ExJxh9Uc3J1mz(cJJHEsfQ!ddh) zN_o<0iUWnHc@r+taPv=8ZR2P5^AFOQ-Vu|#sOCxd)Gjo2m6dW1%6p~8hZaEsY`pg| z0lu#T`!2VzZf430Wx9Je5}d{lFRwM?mH2-)-rv-*1gLkJP5eUSzze^g z@iL#W9G#dDOhok2g8K+TqYCX)bKf4kt~pne+PS7wjZ0hv=_96uol5F1QuisoXn=pN|cJ1qW+Y)2g+XwG}TzvI)C2z=f81vbJImI>v~wZtpK$Syy5 zbgfhDS-%*Y8mzrI>k$1G8s3vAq0-E`H5wd{VsH@!-&_w1Tc$ZF)!FAEB5v_2JV*Wj6-f3K)&t`IR8Qd-? zE0&vhy5EeuaTm*KHB1MCvZ#8Y1HV^69`l)LxuR5prBoz;ReBiD3%?Z z^;IwN+?*3HbN_gy63dM;+^~VIIRGY5i&A?;abnfBvJEczG(~w|pJ9kQH^ZMO=)+m* zwThl%sJ2+QJCKECBLg=P;ry$LPj9)dgs@8_g5{D#RE{5M zL52uX$%Bt%?fjPkW?Yi%wB8t&W=Z@D|XJK^Q% zt>`s&wy{dAg9LQ|NhyCcadB_XaG04}_G6*T>+>7!YCbG7-h5)%U9PJwrghmIov!T= zoR*5p9f#Z6nrVVCcOFM_nVk-Xp<=5@|5oT~=A~H9PXn>QX*`ki(}$+j(=&^yYkdn( zpYO?WrHWbOJ2cnvz`86W8uEku6yk%tq2h2d<(6K2f>eQEa8ZIZw=eIQ@of;ah{TQH z;^3O2=Qre|mw?l0z??AaQhqmC30Ti_7M3Yi}@U(|kXx0YLkZwUMm7Art;aiHraqzJ1=MGjS zou0~P^tedV+yq`Vqz&^Oc{@*gLY%7%@Iw7x>_cAvTCtnmPW*i7{OeoI$EEUfYj7Qg zM&5iu?|}de(tJNr$|+N5_**z!p{r`gaP$wZC23`)q>My!O#F1A>JLwA&9X1jzmuBb zY&tpz!zViPLONGYb41==RCaG{*NhOw!=Z5pTUTTEjZN`fL*;F{C-+@+ zF^S`S(xGsa2TSZ=4ll7jnYdK^2NkOM;wHo}L|uQ*)+2H|ExqF$g)Yn}^!kLYQ=Ohb zSn0*MosER>QQPwXY*z~u`F+NWcOA?8g`a=lhM0pTix_`w3e4Qv&qlu6e6E z-VrX*vGX2kM~st-&_6x}ixH!^tG9|}e_7C|(H$$uYAifR9KTAgGF|Ubf>Uf`D)O-~ zm(c%m7HaJ(tgoR_O7vV?JgFgxAWw%ARn)K%*Ha@|KIcO*_VwfF{D!+-16y!CYlVo_ zr0tt5UI0l@UP~6H*q(};a%JC5^>@{wF6jWC(>*B0COG^LxJs=Or`v|3}j;uC%^A`H|o ziH1hsnPmtwaddZ*=`6dRj(iSG3BlGI<7 zlHvH4#qD!B2eLRXi$mj5rg8pviZZ(aO#x~gdUx9FQ7VRpurxV|u8*iE`5R4V*^Z6x zq&_%4w0!?wSJ+r`h}1y2ded%|faa{*oq!Z|C>1 zm$&`nQNlNb2X<3BsE&~{X1=6|2s77k7?;&wgmyUXndV0g%O!@!r2M7TsC}FH$tr3$Nb1_S zSvrDM*GQxpYZm?L1TmL$PY7?vlcV%I119b1TP?b((g+n+~$&qzyi zpd(MWFb>}Oc2XPCv|6lGwOE@Ey`_-*&Of85wYsqsYh%u4L6?^7SC$}S7< z{(--7Pv796YwiOb%_^Pk)}QHGJh%|Qx@CP|=}6`&+4mLGafD~PfN*oo0X+6b!fpc( z@M55`w*}FV&ypQZgaWcZPY{Mqqw;vaOT!%PE*m;^<$_l2-$RnIBiiTWW~IX`nWpnv zC#Ihni(sX(gy#?}<7m(KaQylQcm11-io9wfC_mWi8g2ko=Cj4p`ANEFSYONa?AnYU zm`|v@xJnO{n!3^!`|yA&b*J z?y5a-2(on>W?$WQrY=pxzN~pfk6OIfl}X0`Bl4+@)*|=!p1G=7tyhgxxeBrE=8A9a zjGcg@r4BJ)MTi)4D)EZTcgAGAALPH!Z6PL(=qh9FRa-}=iYcpO-SOD1t?TCzq0j-_ zYhJV*w=gNM*Uv}kRhvI*KAOFpNwd4|5Nds{{=RrR=5PW%gI6j3D|?1qC#Art#G9jp z(p)Z&nUu=T(|a@VIN!#TknjKk637l2Ij}5G?=o zLNn#L6CfbUqJm2$`oq-qjfLZVfK))BqhWv4yRHE;zc-W(UA6VRqja|bo)Ct*XqgnxvY%p-X_vz(HBSeo$ME~uc+vmH zlx*=IN;ethA78~OmWX`mer7;vZQ0jHg^>%OPTY9+I>^=ZNV68{qtPL$@*~5;D!@@M zzMt~g35pxaCVI`|+i&4!6Xs+Htdg)QA%Z>yDXB2r@)XPXUr@k5OPkq$kc`Q8YiyuJ zpXMh*x^g8lC%TnUi;*$CUP}t(n-&nK+ww-8`hh!FhD+m~zLCilcDjNzZCF8xEuL-L z;L|0TrPN(~z*i#nHc*;ht27cl-lic4C8Cg@%Wp3qfS;?*E;;N zL7u6pA)*vBzNUm+Rxy4OGmQv=-Px+aLzAup;?&Jov_m=!)0!A*9I7_{Yv|}9`Xz&h zMeIcEj}4A9-^#emds3S!V1zPKG8P%zylYoKL}iJKuHqA$B%}WzXq);2CR?`6I`BpU zzGI4#VT40+^|AlL!`-xGQJ$vJaLqBz0e1)$Y!=ZqI78r^o%HKo67G&&;xA5`Djt#8 z5hTxxRml9PK7tAAlBku{3P+I8@Ic7d0<(PyIH4^}nwO$Y#Y|z@Iarvy&Ldn1 zr0D$JRY&tdNQ5v;dP%iX3)>)QSKwQWMQIn6yc+9qt$DzB?&>4(oC+dd>iiL?Wb|uh zw}P4NA&$@rv_v?v>yL0_458Ho^&dQ2n1P~r`?|&CW9CeVE+8;eYNHkaM5Q$Mx^-Yl zWX5r~-}@n=eqH%{?8p(%Iq9g7#?tH2f{CG%1}@oDtE#*n|>5V};L zZjsoPm7nAbcjUw_wC*q__n>@QfRVa<`d~Y9km2|kD1_x)0KzE7{mvE0eV~%XJqJ#A z)1GYl#596x@f$KAOzHNx2vy!4!~>Fo==Pv74lh;gwRV!x()AVU<_IJ7j|tFPiP?++ zYjqacWTuX_Hhb1!TA(A$!W1Y551orOBz1A!(Q}snuSKY8wF!wf!E?z!ryG$TiL!XH z_Mrq9VEioD{KYg(6N{yD^bfPO#jzOW1+Wl65sSfcm zLHj$C{pIm-={D{*7*5|`d1DGqptvu$a9K9CkX&eVnL=*3n%I^5kX!K;9SjGZq>2#> z6I0c?ufEoO#F9G$_6urXB7y%30|Te2n0H56Q6^+1ApfY1qXnsRUO1O-E)I-moiUP> z)iXmC0-p`tac6x*uYGPHnCxo3-($5#hV-}AX`3fYh20Rr5btB5B}OH}jc9%M-pc{; z_d%7Vd|YorBIu9s!(L;HkaXT}qMi+xYa~+I1Il#{_B=|4a}>AHZgED&bGzfF=N-V>9?L@AW=J`fUAHQ~(w)5(jvSUcSZ((2iq<2^-W)wVPs-3Fa= z2*07$g0L=p9aBv4i;erPCSgD`n`G)V@+;vPb{Iyi+r89q3M5zQ!t$fR5O>oW%tJeH zV%%}=IE3FP`$cFOKh!`BZ(H;a5v>d0G67z-i%uoCdW99RP0OZSF-+$@J|-jrqc{rlb^ z(eAU?7IHs%1wHC^x}ZS>>)TVbVuYR<-K#a(FydYHue?cpi^leLa@ayJee6S?nlA>G zlq1w?N1B_+>dGjVN?=UDi9`7Zbvi3O+pF%#uCYVnxOLD}Vx0W0x025_AEtO4B>5(# z*}D%k>z|m=Uda}2RN!_WI`hqzB&SP<5x2M!E2CCiEtTz1`=p<^0Lw(lFK|Oz5znRa zE!{Dp3)Q_bGHXO!x%Fs;(D5Ul;--a@-M*1gwdm;?)*FeR_i(?CgYQ8E#G3&C`Evvf zvBX4tc1!Fr{PTVOuMrlwIak!}8+73bhqe<1S+lkE-WvtAxldNIT%zopA#WK0+hCA} zq3xl)*`#|Hd=g`x__3-DQM7=mJ}$~%ECHj&uB$iDG>gO9Nae~qJ=R#d8jCr&d+s1) z!B{ABCgzhQohf;OHOEDTcxtztpJnDZM_RDMuntT6`Guo%|LTgG|MVN>i3>u*eK)~=RZ1DQI9VNoA4HH%+zW*KT9j84HPcCS3Rtf06t;**9V7CkKo*7lox^&>yU zK+H3z*Xghx=sT%{`ljv#rRi3(wd2M@l289nU^AvQihx!~t;=8jlU1qCRq>VO> z8zHBMgnB=jzbgKIV9%Zg84t%j7nc+)nkD*BDvPNTFtRbk`ihKR3&DL7jv^Gz)cHh> z59n^R<6HPrx1i{Pv@t^5Ii0dc)1 zhwz~z%0f}N<`fHEHcM-2MtLB%`Vr!L^7l2FSWnt)NEOKI${&cD&36r;rV!qAgWg4B zm`SDlSYkzh3w5u8zw1Sa$9$2ZP0*pJ@iX;J;s=;B&-JKF++m`gqw=uV>-kX*q;VYy zFk`_fkjUnb7G}&>aGWQqy$2Up?kW){Jm9GO8j`+DqE{n>G0NdK-H(_sa(p|U4|n6q z-D3sK?Y)KzsXLoiFA97T(WD`PO1wy`%s7JI-N+pjl#VHzge!WOaGd;Wq;+U({aS1U zS$)`9KO}FuY%&^*i6tguF2N1 zA75t$3Hd`5g+AQaaBptFU)G1 z$u)xTx#n~r@Ht;*&M`JId;Y|?cb_Y_#TDIvK!lsc?a^XDF2q_QBz(x$!EJlcErAm+ zS|Os5^fCGZo%{?b1f@tWoFxzeR?hIzVe!|AkAu7uK9>i#w~Fp7)t4s&7KmP?rkQ*w zvq`acV3;=o)&@=X>&Hd^=tHU$1YK&v$K^F4vxi@={3a8|4+JQz1r2^f=}c1RD4#vg z!D9+y9`~yf+vOSQd9^6FUE|b+=r-dEji(+1CrW`}&fYsVB0y-Q4^b>|tQ0~mBU&x^ z{e#3r_mn7hrrK`eUM6u6ljIlMG8u`iI%^GkZT794I>sv*AfZ`&xdJvx$z&o)te0*2 z%3LsM_kf_7-7ygK5LCyy`Io)g_8D_s;s0z&M9d9XYB@4KWWJ*O31Tt7r2 zFvY_|lH!By4UIbJC-1 zPL>OpnS_QW|5G9z6gNt|n8JLC&?!)l)F)@Dn`Ehyk(!nkE_ki$+P*GLN%<))i~|M* z){9)!|1Qy;hoD8#FvcnRfuzU*c4yN;o@i3iXjBTDyA_(F!9`4{sh<|{0$*= z(xLC3KEHS3=|pGEsB$7$n27!?5T#_}k?R2;qyz-y;s$g>rbJP1V(4G`Ec}+lGFu^T zn;b=S*CEFTI2OC%=U}suDb&(cGcSiNyw*F!u5AlY5Cz+cvMrzvuM0|hMCJ6@VF>g# z+j}Hji<#$1WFC6dL$c*lqbsFZx~(44J`qXMjI;^_(Kd52jgNPdFWuhPM_m({y4DMB zi~*><{c~c?IHEBNWWrs?%HSG~DX_a+>3R5Uq8+4i>nsM6s2;I7Zd@{lTusJ}kma`( zcf~W<9?%@8j;;>55pqvH8kQ5ACU*$Gmsg!Ka539FEu&X(d#~V$i{{!c1ZU1XKPDyh zcRsZl5&NBe;rXkv+~75;e6^O!F6h;9@tLbDVy8dbOo4N8*aOdPI7UZjGqZ?&M#mUZ zQx})_l&iQU9`SVm_?fx=NONRBwooUGJxPAg4w|D*){xlm`t-9_#$kc?ke6V-@&%Y}Qk7in`0o>3Nbo;Okb| zu>}w%aK`E~XL(JEurYm7N?%c^^DkXqw(1l)7iq51WmP`ub243<{kz>ysDx=2;O7j2 z%Lfl>#r6TqCWacDxib$kxCIuDClhjI+s-w`%tQMyG+?13iT8Z%{S^s&uR}~VY$l@X zQv}xtO9zL4JOdhbu5EA?H+;O7Ni+A%1ZU%;@UchNZe@GMF<1n#(hE6$5tnCWucK@a zax;uU&H!_^sn&V>6HnNnz^ijlZ$X&cofV>8GrUIYhm&egaD{nM6XoO%&#|bRUZpab zo#7;|bvHhX#-$dNOCh%JVBfco@OL7$#VIY2GBqrpuLMueurKX~9of-!Fw?$d)Hihm z?Y@3|*Bo^j;P?rNZ&Iy`c7%-{yQ`lyb*tbnI0%kKjoD|j*oE@CaR7CO{dQPfi}M^+ zh`?eo9-a6e3H1`((($ne`}@YCnj&2BX>J693_-1Vb0U;E&89cAdMZYwTzxVR9Y=rOl89>tz!s=TW$goJAKZ-JbPhT?5V6-Yry;fK zGY`g11}S;Vc8fp3f0?EmDRr4HptW%#pnk+ER%Am&*9qD7Ab@a}rA308&KvY#%B|@W zgTKk~x8FdpHC; Date: Fri, 6 Mar 2020 05:58:59 +0000 Subject: [PATCH 04/87] Readme formatting --- docs/google.md | 14 +++-- docs/images/google-cloud-logo.png | Bin 13226 -> 0 bytes docs/images/google-cloud-logo.svg | 96 ++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 5 deletions(-) delete mode 100644 docs/images/google-cloud-logo.png create mode 100644 docs/images/google-cloud-logo.svg diff --git a/docs/google.md b/docs/google.md index d2b0f51..d3a65f5 100644 --- a/docs/google.md +++ b/docs/google.md @@ -1,8 +1,11 @@ # nf-core/configs: Google Cloud Life Sciences Configuration -![Google Cloud](images/google-cloud-logo.png "https://cloud.google.com/life-sciences/docs/tutorials/nextflow") To be used with the `google` profile by specifying the `-profile google` when running nf-core pipelines. + +![Google Cloud](images/google-cloud-logo.svg "https://cloud.google.com/life-sciences/docs/tutorials/nextflow") + + ## Quick Start ``` GOOGLE_APPLICATION_CREDENTIALS=.json @@ -10,21 +13,22 @@ NXF_MODE=google nextflow run nf-core/rnaseq -profile google,test --google_bucket ``` -## Required Parameters +### Required Parameters #### `--google-bucket` The Google Cloud Storage bucket location to be used as a Nextflow work directory. Can also be specified with (`-w gs://your_bucket/work`). -## Optional Parameters +### Optional Parameters #### `--google-zone` The Google zone where the computation is executed in Compute Engine VMs. Multiple zones can be provided separating them by a comma. Default (`europe-west2-c`). #### `--google-preemptible` -Enables the usage of preemptible virtual machines with a retry error statergy for up to 5 retries. Default (`true`) +Enables the usage of preemptible virtual machines with a retry error statergy for up to 5 retries. Default (`true`). #### `--google-debug` Copies the /google debug directory from the VM to the task bucket directory. Useful for debugging. Default (`false`). -## Google Cloud Life Sciences Setup + +## Cloud Life Sciences Setup Please refer to the [Google Cloud](https://cloud.google.com/life-sciences/docs/tutorials/nextflow) and [Nextflow](https://www.nextflow.io/docs/latest/google.html#cloud-life-sciences) documentation which describe how to setup the Google Cloud environment. diff --git a/docs/images/google-cloud-logo.png b/docs/images/google-cloud-logo.png deleted file mode 100644 index 07c9a325b254d93f3fe4e94abb5cbdd9162afd92..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13226 zcmZ{LWmFtZ(C#em?hxEvg9ZyO!QI^@xVuYm4=%yoC1}tM9^5SicY=SL_ulWE`{$lL zXJ=-&d#0zlx}K`1s-u-ZNuweWAprn@Dk~$Q3IL!R$oVG(ILP;Q0N3;+U|0D$lOwNph1@&}x$ytD-H z_U}{B^DPAcIQwNK#MHf3P7S>M)HLqi@}81wmEYs5zsGzhZJ`2#uReZsc4)m;peQ4k zt$ExYPjv_$LiN<2e$S$qjdNeQ`-MDb_x&r*y}Q1su{kCNJ&Qw%=hS(EX4w048bw2` zjeD-YS?}w3@AEv*U-DefI(}maP%I%g?61#Amtzt^570xQ!a1`CiIJOg3y0+a>Oc<2 z9_~_#-Ct>IihAxhwVjBd@sA8alYuZqY+8jA=z_`Fq+2V}u3l9q{>Z4;p)Ew9Fl+pts3YF0?dq9pjU|kyJuiE?tw!fxd z<4?a^T7^t@lL6`*>IRdeChZ`AD4!w>skt1#KanwC-N zgjUBRs;j6=bl^qM>!d4l?3s78R^UKbnn5-Ag?%v6-c>g=74>ea~W)aCl zvE(Fkl6wu&QEbbBW06eYivJ^!PArL>q=&HG5zO5lR*3_^8Q@}B=Kg%L`M(YByu~-U zZK~geq=WbhkjEt+DyO0Zzpb^N%I0LKff-Ey6Kf|Fif4z@SE2VAnM^CQk52M_zkG4d zh}`LTqqt3nkKGBg0t3!3fV{kJF#9offfA!yC*J!|LzcW8P=Fq7h_i7%x~>AE*CT+ z3Ek#|&-TdVq#KNx&uaW@jHrdxVqUNP$1b>;MicC0=A*k8Is zuuCNptn(2W0+Hb4d$~%@2PELJ9;dJq-q}-3L;`omG2(wa?-ABMT-Ek4U(<(l2K-U{ zaJGq08~)Q$ehhrXYXHTYcK?*SWitk!FN|(UqkW~<%JwatD3}-yFV)W0 zJix0>K5r|h)mH-pq*=FDm~bbjsUp)z1Mca~7vWr%c%P~=jh<^eNE9=dkoVm1NQhMa z(yKm*h8#|WkxF#Sb8=vMU`>!|Z=Z`>Scqiqa3pR{3sI<|Rme%vjfJMwxeVa*?>~^*?%s)$tMC+Tub7o&?pAQI`76on5VATs@JpXOu+y6QW6*1h!s;8-GP}Zbdqi z#R4!1;wgl{K=9085h(SL^?e=mVI*Mcm!^|pcD%;{whw{bST4jA&?Q~rTEFamuH^QP0~xKlIfahLhN2e1ly28=O- zx&jG#YriSfh>%HWVX?no+4NsH_geE*s2cOT&(3j^cHf`P@nJnj!=|!|c!H6+)Y)DcNtGL|zy{vWL0wT!e06CSk7=3vtKB-I7{^eJ^-b?Y)}=_ny?VeXr-; zYN*DbAXs9$ge(Q{Jv8?KIj2+h^-a(zA*q2oM}j7v9 zmreoz$Cy5!()F}J-_tKO=;|686=ls`B3P6+MorSUXwXe4r--|Xc+RBld!c0UwpV>L zWX#(qCo?&zD|U!}Pv2gKb`9hHvxJU*98M+F5XqfCzMBdL+jIJeRl0`iaoX- zHjSh~0zaO<)gr$UI~_tiy`tX%vXOU{L)lhhY5pd;$$>F1($)DMzT>SxE zYfm3m?oIJ~$*PvYRB#kDNzWmY4o&Ju+X9kyc zc7PvV_vbZ(2?T8~Q83AHrG2EqZMr4Pc{bz2=9!&MyAsHN_H$N@`l1vSF;2 zcKmom)+R=%4BdD*#C5wdSKP@0+xH7oyAG*07+v()R-#2HJ-}zZvX$h*cX771V&mwV zO4AQm=UQ)fUI|iUG|P~(P%b4nk1BGJ~LbMY8@8z#uHh5qP%n}&fcGDEwpdA|VP z*Vi#*k5EQ9J3Fkv#$a2rOTYHJLObuU*)7;D7?btROIcIXGWp+NzRfMC;QHXMEOWPU z?oJg3onMViOr+4RCid?As!xl+HdARzSYS|Cwk)JhnFaw#2&|Y{CgXfos6P9Fp{ba9 z*fgJWUoZFp7BsdNRvT^0cgvd4MtApl69A)P>pNP3UZ(Ct1m;C@DIY^q91S?R^vnr} zN3dHEGcOI0)HWPxGAPh#!TG&$UJ2&YGd%G8pqzr;zAQ7U(aapqxy4ox-;4)e2M7op z$JE)XMLNN*bY2jcVafX87@GP=WsLZ?RJzxPmEw!KC%=#g|C!9@$bF#cT0b)cYU4P* zScQT%{aj@sKu6+YsNm{Dp! zG4&l-;jwAsLLoR#5#6C{_bHig5CPwpSrz@kKRnIC!VdgT-@I&Awq$Z|p`lp+^aYU8 z^qq%gXwwp}*E6PaQU|8zjrD~@d(Pu{4tM=Q8J~Ijir~t`prAu)-5t=)~q`_xS(DURutWlD%7@7T3UkRpLFUxEoVyJJR`$C(ZCC*(lCPO9EHq2=A@)w zPLYEe^@)T^Bm#$hS}R4OaX&59W_@ozL{ck`te{%SQ21cHX?~2&ZtrTSOgD~L|Fb+4 zhL4C4{`}o@MO>-y&X#!YuIv?w?^-Fy>%V-vl3 z9JGlMsHdu05k3NqZPr?)xg@AWx-tN>7B zuaaOGw9e(j_TBJM(n8oQ4-f%VqZW@rka7Wg(7P5aNFOagB;+qkMGQU~vEP^!sH=SebKZ2=Q0B7I~_`6A_H%o zF%icm)bMos@lR|ta%0+{EHthDC-UzNJh6J4X4`~Nu;`r5r1Z5h(S*kO3e>EvT|ycq zF;yY`JE$;^@O>&wx8Z{opk|fu+O;gM1acd1ge|U}bXsHTx9sB^FmB`7nKAp4 zlx+N=YCO6j;?KaF3IE_xS4*5YeDO)mel=pPqyz3_Xk+e7Ddc{wI^C{qc*tbEs#8qp z`L<}#yi$zQ-oFc?!g@FI#URN{^x5QkGZsz=aqDVGKYAOB2FGS-)5 zjI(0!kdMJ$rNdcJek|8KZz-B2BZrW^$SeTM87>j567R6ec5MepZAO20oIU-O;nrS) z;G^|6?`$xhvu**M^}dFFV*3gP2lQ@UaYEUCPd zUOKYAf{Ul5=5BA@hIPhULh-OV3#ZX*yZn7;H|mJ}Oh9(FD0XD9iar@;KK~Wcr&Bb} z&wL~{Xc;4msdt@ezS!}m)8dg6*v-qevhV=q3Mqb}bh>UW=u#(FPw2U*kw@QJ-Hz#+ zA#RvEX8107ROGVNA<{iH74o!c8q5K$1QB5;>3ePVhe*BI0VcwF;-wRt>Wh(%$&j!R zj?>y_S;z%|vX*M?)5EAKkP90qBlF2SPf!#mhAJbldfA_z=|(uH`c7Txbn;mm;f6j) z9eRaWvGVWqA22WX=8uHq_oQ>??(Z{}TUXxYY>M(mtnU-j4g7@(i$!6V`eJ_YPhi!x z%qbJdNRRc&tzUzmVQ>#=OG)qIstxKo*3VEZC1`GX^v|B99t#jC+dLdEr>oKl#bib-B-$KM^lOt_*z zPhe7g{uFXu!;9H_Fgnr8t!?<|w>I>LkuSer30lSm`ZGxxR&P#@7JFf1_g>RNB0rZU z@lt~DsRAB4bo;=T<-Jxko}wpi-c@GrQsS+s;xc!!^#OAt`A$wYI?VGQf91R-2fkMbC=uwNxAeTEIfN^)I9eg( zLV}kY{o(NOpv%OfjP9iNxq>3_Pp=uaReuHeS)DlP!Rb&Q9BT=tCm~kqOBl>DGH1_f ztAiiTG8$?~rsB!x6otmnQmur>9nElYaTzAS9GglUq>rJHf_htYV7cxNnK4%XL4M6+ z9FPU8i4YYZ--a=KsZFK6#wlJuDbAT<~aD8&5$ddY+CDKS^7>BY-&2vO+2 zVP4VzrZj3tY~g+atH&Q4!WXN&iz)ww544>kXoQl0;c!6)>|fushAeWQls;wl{PjH> zIh%9ObLaL*u9=?CLXRJU9!f?tYy)0yM z?!oEQINgEj&mY;Frs_)?gl|l#B6q(0%p1DsNFQ>rvl-t4;D$ZZ)JZ+qLkFP)II9ME zpMe7qpXRrkPze}UkhKq`MCLjC;TFNl7(9kL#!W_b+=fv%>ei>OIkV6{{rF0q-t7`G zy9Fayi7A-_kFJ_q#s`r0Z3Y1z=5-4Mz>I8!RNt>_@F|E2ONe$2FV$0$Mi-JxM%j48 zNcz|Fo4|zrAO^Zowk*0PxX|K$ixKVT2NQOD^8Asvc!d=rdT#xNgQEVsV6B2ILmOy1 z2KSM-Vkf!mz981vdn~Zq@VugG!Lgfo%bGU~hU-~UQc~LeuIu<8sbJ}yFg%hIeK>WA z(|`5!q6gB?%E@?x3(kvk|4I=wEchG9wJH~b?e6b0(l(6?X9;DI`%8|;mqh5u&f++) zykmRR(MszIC`UV)mt99+@B(je#@5N0W9_x;_(TIO9L750ppyxO{yWf##?>hi&4l`4%xQZ)(u7r`&fR zN6BIa@+*HLihlL>789GlJ`qT>gmQC3aliX++~-+_mIV!vUgHNE{khNm*!S`DcEdFn z)(VPG@pqB{gYzeLWf!MvrcV^T=kHX9_yv5?F{6D^mR?kg&p{+&yn}99+w5@7P^p^} z2B!23brVF6oP*xLeN*q=H)>H>S(QKTTSNa%hIozhpgP4EuSe8EP&Hs>a z3q2Ds?RGird#zp2#t-hJ64ApDepjF&kYJ&6q-VI>yljsYY|4;K^Y3jj1M4y2U%pqF zF$%+p1-%tj{RVO2f$v#j0x2ac`#bv!==KHHzfpJH6iu1TltOjx1P>hjQt1(iQ^btX z|5{G-!CdrB55yeDC@r?QV&9rN$(TaBFL2~QY?+7q)#zD_@>CN}4iD=vI5P~bXabcG zagHUuZ9q+}EJ7su)_P^=(WxedcoOy+H7ZZOXS}ExJxh9Uc3J1mz(cJJHEsfQ!ddh) zN_o<0iUWnHc@r+taPv=8ZR2P5^AFOQ-Vu|#sOCxd)Gjo2m6dW1%6p~8hZaEsY`pg| z0lu#T`!2VzZf430Wx9Je5}d{lFRwM?mH2-)-rv-*1gLkJP5eUSzze^g z@iL#W9G#dDOhok2g8K+TqYCX)bKf4kt~pne+PS7wjZ0hv=_96uol5F1QuisoXn=pN|cJ1qW+Y)2g+XwG}TzvI)C2z=f81vbJImI>v~wZtpK$Syy5 zbgfhDS-%*Y8mzrI>k$1G8s3vAq0-E`H5wd{VsH@!-&_w1Tc$ZF)!FAEB5v_2JV*Wj6-f3K)&t`IR8Qd-? zE0&vhy5EeuaTm*KHB1MCvZ#8Y1HV^69`l)LxuR5prBoz;ReBiD3%?Z z^;IwN+?*3HbN_gy63dM;+^~VIIRGY5i&A?;abnfBvJEczG(~w|pJ9kQH^ZMO=)+m* zwThl%sJ2+QJCKECBLg=P;ry$LPj9)dgs@8_g5{D#RE{5M zL52uX$%Bt%?fjPkW?Yi%wB8t&W=Z@D|XJK^Q% zt>`s&wy{dAg9LQ|NhyCcadB_XaG04}_G6*T>+>7!YCbG7-h5)%U9PJwrghmIov!T= zoR*5p9f#Z6nrVVCcOFM_nVk-Xp<=5@|5oT~=A~H9PXn>QX*`ki(}$+j(=&^yYkdn( zpYO?WrHWbOJ2cnvz`86W8uEku6yk%tq2h2d<(6K2f>eQEa8ZIZw=eIQ@of;ah{TQH z;^3O2=Qre|mw?l0z??AaQhqmC30Ti_7M3Yi}@U(|kXx0YLkZwUMm7Art;aiHraqzJ1=MGjS zou0~P^tedV+yq`Vqz&^Oc{@*gLY%7%@Iw7x>_cAvTCtnmPW*i7{OeoI$EEUfYj7Qg zM&5iu?|}de(tJNr$|+N5_**z!p{r`gaP$wZC23`)q>My!O#F1A>JLwA&9X1jzmuBb zY&tpz!zViPLONGYb41==RCaG{*NhOw!=Z5pTUTTEjZN`fL*;F{C-+@+ zF^S`S(xGsa2TSZ=4ll7jnYdK^2NkOM;wHo}L|uQ*)+2H|ExqF$g)Yn}^!kLYQ=Ohb zSn0*MosER>QQPwXY*z~u`F+NWcOA?8g`a=lhM0pTix_`w3e4Qv&qlu6e6E z-VrX*vGX2kM~st-&_6x}ixH!^tG9|}e_7C|(H$$uYAifR9KTAgGF|Ubf>Uf`D)O-~ zm(c%m7HaJ(tgoR_O7vV?JgFgxAWw%ARn)K%*Ha@|KIcO*_VwfF{D!+-16y!CYlVo_ zr0tt5UI0l@UP~6H*q(};a%JC5^>@{wF6jWC(>*B0COG^LxJs=Or`v|3}j;uC%^A`H|o ziH1hsnPmtwaddZ*=`6dRj(iSG3BlGI<7 zlHvH4#qD!B2eLRXi$mj5rg8pviZZ(aO#x~gdUx9FQ7VRpurxV|u8*iE`5R4V*^Z6x zq&_%4w0!?wSJ+r`h}1y2ded%|faa{*oq!Z|C>1 zm$&`nQNlNb2X<3BsE&~{X1=6|2s77k7?;&wgmyUXndV0g%O!@!r2M7TsC}FH$tr3$Nb1_S zSvrDM*GQxpYZm?L1TmL$PY7?vlcV%I119b1TP?b((g+n+~$&qzyi zpd(MWFb>}Oc2XPCv|6lGwOE@Ey`_-*&Of85wYsqsYh%u4L6?^7SC$}S7< z{(--7Pv796YwiOb%_^Pk)}QHGJh%|Qx@CP|=}6`&+4mLGafD~PfN*oo0X+6b!fpc( z@M55`w*}FV&ypQZgaWcZPY{Mqqw;vaOT!%PE*m;^<$_l2-$RnIBiiTWW~IX`nWpnv zC#Ihni(sX(gy#?}<7m(KaQylQcm11-io9wfC_mWi8g2ko=Cj4p`ANEFSYONa?AnYU zm`|v@xJnO{n!3^!`|yA&b*J z?y5a-2(on>W?$WQrY=pxzN~pfk6OIfl}X0`Bl4+@)*|=!p1G=7tyhgxxeBrE=8A9a zjGcg@r4BJ)MTi)4D)EZTcgAGAALPH!Z6PL(=qh9FRa-}=iYcpO-SOD1t?TCzq0j-_ zYhJV*w=gNM*Uv}kRhvI*KAOFpNwd4|5Nds{{=RrR=5PW%gI6j3D|?1qC#Art#G9jp z(p)Z&nUu=T(|a@VIN!#TknjKk637l2Ij}5G?=o zLNn#L6CfbUqJm2$`oq-qjfLZVfK))BqhWv4yRHE;zc-W(UA6VRqja|bo)Ct*XqgnxvY%p-X_vz(HBSeo$ME~uc+vmH zlx*=IN;ethA78~OmWX`mer7;vZQ0jHg^>%OPTY9+I>^=ZNV68{qtPL$@*~5;D!@@M zzMt~g35pxaCVI`|+i&4!6Xs+Htdg)QA%Z>yDXB2r@)XPXUr@k5OPkq$kc`Q8YiyuJ zpXMh*x^g8lC%TnUi;*$CUP}t(n-&nK+ww-8`hh!FhD+m~zLCilcDjNzZCF8xEuL-L z;L|0TrPN(~z*i#nHc*;ht27cl-lic4C8Cg@%Wp3qfS;?*E;;N zL7u6pA)*vBzNUm+Rxy4OGmQv=-Px+aLzAup;?&Jov_m=!)0!A*9I7_{Yv|}9`Xz&h zMeIcEj}4A9-^#emds3S!V1zPKG8P%zylYoKL}iJKuHqA$B%}WzXq);2CR?`6I`BpU zzGI4#VT40+^|AlL!`-xGQJ$vJaLqBz0e1)$Y!=ZqI78r^o%HKo67G&&;xA5`Djt#8 z5hTxxRml9PK7tAAlBku{3P+I8@Ic7d0<(PyIH4^}nwO$Y#Y|z@Iarvy&Ldn1 zr0D$JRY&tdNQ5v;dP%iX3)>)QSKwQWMQIn6yc+9qt$DzB?&>4(oC+dd>iiL?Wb|uh zw}P4NA&$@rv_v?v>yL0_458Ho^&dQ2n1P~r`?|&CW9CeVE+8;eYNHkaM5Q$Mx^-Yl zWX5r~-}@n=eqH%{?8p(%Iq9g7#?tH2f{CG%1}@oDtE#*n|>5V};L zZjsoPm7nAbcjUw_wC*q__n>@QfRVa<`d~Y9km2|kD1_x)0KzE7{mvE0eV~%XJqJ#A z)1GYl#596x@f$KAOzHNx2vy!4!~>Fo==Pv74lh;gwRV!x()AVU<_IJ7j|tFPiP?++ zYjqacWTuX_Hhb1!TA(A$!W1Y551orOBz1A!(Q}snuSKY8wF!wf!E?z!ryG$TiL!XH z_Mrq9VEioD{KYg(6N{yD^bfPO#jzOW1+Wl65sSfcm zLHj$C{pIm-={D{*7*5|`d1DGqptvu$a9K9CkX&eVnL=*3n%I^5kX!K;9SjGZq>2#> z6I0c?ufEoO#F9G$_6urXB7y%30|Te2n0H56Q6^+1ApfY1qXnsRUO1O-E)I-moiUP> z)iXmC0-p`tac6x*uYGPHnCxo3-($5#hV-}AX`3fYh20Rr5btB5B}OH}jc9%M-pc{; z_d%7Vd|YorBIu9s!(L;HkaXT}qMi+xYa~+I1Il#{_B=|4a}>AHZgED&bGzfF=N-V>9?L@AW=J`fUAHQ~(w)5(jvSUcSZ((2iq<2^-W)wVPs-3Fa= z2*07$g0L=p9aBv4i;erPCSgD`n`G)V@+;vPb{Iyi+r89q3M5zQ!t$fR5O>oW%tJeH zV%%}=IE3FP`$cFOKh!`BZ(H;a5v>d0G67z-i%uoCdW99RP0OZSF-+$@J|-jrqc{rlb^ z(eAU?7IHs%1wHC^x}ZS>>)TVbVuYR<-K#a(FydYHue?cpi^leLa@ayJee6S?nlA>G zlq1w?N1B_+>dGjVN?=UDi9`7Zbvi3O+pF%#uCYVnxOLD}Vx0W0x025_AEtO4B>5(# z*}D%k>z|m=Uda}2RN!_WI`hqzB&SP<5x2M!E2CCiEtTz1`=p<^0Lw(lFK|Oz5znRa zE!{Dp3)Q_bGHXO!x%Fs;(D5Ul;--a@-M*1gwdm;?)*FeR_i(?CgYQ8E#G3&C`Evvf zvBX4tc1!Fr{PTVOuMrlwIak!}8+73bhqe<1S+lkE-WvtAxldNIT%zopA#WK0+hCA} zq3xl)*`#|Hd=g`x__3-DQM7=mJ}$~%ECHj&uB$iDG>gO9Nae~qJ=R#d8jCr&d+s1) z!B{ABCgzhQohf;OHOEDTcxtztpJnDZM_RDMuntT6`Guo%|LTgG|MVN>i3>u*eK)~=RZ1DQI9VNoA4HH%+zW*KT9j84HPcCS3Rtf06t;**9V7CkKo*7lox^&>yU zK+H3z*Xghx=sT%{`ljv#rRi3(wd2M@l289nU^AvQihx!~t;=8jlU1qCRq>VO> z8zHBMgnB=jzbgKIV9%Zg84t%j7nc+)nkD*BDvPNTFtRbk`ihKR3&DL7jv^Gz)cHh> z59n^R<6HPrx1i{Pv@t^5Ii0dc)1 zhwz~z%0f}N<`fHEHcM-2MtLB%`Vr!L^7l2FSWnt)NEOKI${&cD&36r;rV!qAgWg4B zm`SDlSYkzh3w5u8zw1Sa$9$2ZP0*pJ@iX;J;s=;B&-JKF++m`gqw=uV>-kX*q;VYy zFk`_fkjUnb7G}&>aGWQqy$2Up?kW){Jm9GO8j`+DqE{n>G0NdK-H(_sa(p|U4|n6q z-D3sK?Y)KzsXLoiFA97T(WD`PO1wy`%s7JI-N+pjl#VHzge!WOaGd;Wq;+U({aS1U zS$)`9KO}FuY%&^*i6tguF2N1 zA75t$3Hd`5g+AQaaBptFU)G1 z$u)xTx#n~r@Ht;*&M`JId;Y|?cb_Y_#TDIvK!lsc?a^XDF2q_QBz(x$!EJlcErAm+ zS|Os5^fCGZo%{?b1f@tWoFxzeR?hIzVe!|AkAu7uK9>i#w~Fp7)t4s&7KmP?rkQ*w zvq`acV3;=o)&@=X>&Hd^=tHU$1YK&v$K^F4vxi@={3a8|4+JQz1r2^f=}c1RD4#vg z!D9+y9`~yf+vOSQd9^6FUE|b+=r-dEji(+1CrW`}&fYsVB0y-Q4^b>|tQ0~mBU&x^ z{e#3r_mn7hrrK`eUM6u6ljIlMG8u`iI%^GkZT794I>sv*AfZ`&xdJvx$z&o)te0*2 z%3LsM_kf_7-7ygK5LCyy`Io)g_8D_s;s0z&M9d9XYB@4KWWJ*O31Tt7r2 zFvY_|lH!By4UIbJC-1 zPL>OpnS_QW|5G9z6gNt|n8JLC&?!)l)F)@Dn`Ehyk(!nkE_ki$+P*GLN%<))i~|M* z){9)!|1Qy;hoD8#FvcnRfuzU*c4yN;o@i3iXjBTDyA_(F!9`4{sh<|{0$*= z(xLC3KEHS3=|pGEsB$7$n27!?5T#_}k?R2;qyz-y;s$g>rbJP1V(4G`Ec}+lGFu^T zn;b=S*CEFTI2OC%=U}suDb&(cGcSiNyw*F!u5AlY5Cz+cvMrzvuM0|hMCJ6@VF>g# z+j}Hji<#$1WFC6dL$c*lqbsFZx~(44J`qXMjI;^_(Kd52jgNPdFWuhPM_m({y4DMB zi~*><{c~c?IHEBNWWrs?%HSG~DX_a+>3R5Uq8+4i>nsM6s2;I7Zd@{lTusJ}kma`( zcf~W<9?%@8j;;>55pqvH8kQ5ACU*$Gmsg!Ka539FEu&X(d#~V$i{{!c1ZU1XKPDyh zcRsZl5&NBe;rXkv+~75;e6^O!F6h;9@tLbDVy8dbOo4N8*aOdPI7UZjGqZ?&M#mUZ zQx})_l&iQU9`SVm_?fx=NONRBwooUGJxPAg4w|D*){xlm`t-9_#$kc?ke6V-@&%Y}Qk7in`0o>3Nbo;Okb| zu>}w%aK`E~XL(JEurYm7N?%c^^DkXqw(1l)7iq51WmP`ub243<{kz>ysDx=2;O7j2 z%Lfl>#r6TqCWacDxib$kxCIuDClhjI+s-w`%tQMyG+?13iT8Z%{S^s&uR}~VY$l@X zQv}xtO9zL4JOdhbu5EA?H+;O7Ni+A%1ZU%;@UchNZe@GMF<1n#(hE6$5tnCWucK@a zax;uU&H!_^sn&V>6HnNnz^ijlZ$X&cofV>8GrUIYhm&egaD{nM6XoO%&#|bRUZpab zo#7;|bvHhX#-$dNOCh%JVBfco@OL7$#VIY2GBqrpuLMueurKX~9of-!Fw?$d)Hihm z?Y@3|*Bo^j;P?rNZ&Iy`c7%-{yQ`lyb*tbnI0%kKjoD|j*oE@CaR7CO{dQPfi}M^+ zh`?eo9-a6e3H1`((($ne`}@YCnj&2BX>J693_-1Vb0U;E&89cAdMZYwTzxVR9Y=rOl89>tz!s=TW$goJAKZ-JbPhT?5V6-Yry;fK zGY`g11}S;Vc8fp3f0?EmDRr4HptW%#pnk+ER%Am&*9qD7Ab@a}rA308&KvY#%B|@W zgTKk~x8FdpHC; + + + + + + + + + +]> + + + + + + + + +Cloud_Logo_Nav + + + + + + + + + + + + + + + + + + + + + From 44cd3089def3f47cc1a53a6f7536c4d203e05fab Mon Sep 17 00:00:00 2001 From: Evan Floden Date: Fri, 6 Mar 2020 09:48:00 +0100 Subject: [PATCH 05/87] Alphabetical ordering of GitHub workflows --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d68f44f..5e9676d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'google', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 8e42bbd7d055acae8fff3ed8fca372979a96adfa Mon Sep 17 00:00:00 2001 From: mseybold Date: Fri, 6 Mar 2020 10:25:08 +0100 Subject: [PATCH 06/87] Update cfc.config We just added some new HighMem nodes to the cfc, Partition/queue name is qbic, each node has 128Cores and 2TB of RAM. The old queue (Name compute) is set to default by slurm. Each process over 60GB of RAM or over 20cpus should run on those new nodes. So added the queue State and adjusted the max_memory and max_cpu setting. --- conf/cfc.config | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conf/cfc.config b/conf/cfc.config index 763dcf0..99917ba 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -13,6 +13,7 @@ singularity { process { beforeScript = 'module load devel/singularity/3.4.2' executor = 'slurm' + queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' } } weblog{ @@ -22,7 +23,7 @@ weblog{ params { igenomes_base = '/nfsmounts/igenomes' - max_memory = 498.GB - max_cpus = 20 + max_memory = 1999.GB + max_cpus = 128 max_time = 140.h } From 52842de88b0d64e2bdbfdf61a833b975879c2546 Mon Sep 17 00:00:00 2001 From: mseybold Date: Fri, 6 Mar 2020 10:46:37 +0100 Subject: [PATCH 07/87] Update cfc.config test failed, so added the the default queue --- conf/cfc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index 99917ba..d1a109a 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -13,7 +13,7 @@ singularity { process { beforeScript = 'module load devel/singularity/3.4.2' executor = 'slurm' - queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' } + queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' : 'compute' } } weblog{ From 2135996b13220dc76b4fe1b1196384307796c317 Mon Sep 17 00:00:00 2001 From: mseybold Date: Fri, 6 Mar 2020 11:12:21 +0100 Subject: [PATCH 08/87] Update cfc.md --- docs/cfc.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/cfc.md b/docs/cfc.md index a7f6beb..acf93dd 100644 --- a/docs/cfc.md +++ b/docs/cfc.md @@ -10,8 +10,11 @@ Before running the pipeline you will need to load Nextflow and Singularity using ## Load Nextflow and Singularity environment modules module purge module load devel/java_jdk/1.8.0u121 -module load qbic/singularity_slurm/3.0.3 +module load devel/singularity/3.4.2 ``` >NB: You will need an account to use the HPC cluster CFC in order to run the pipeline. If in doubt contact IT. >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. +New HighMem nodes are available on the cfc, Partition/queue name is qbic, each node has 128Cores and 2TB of RAM. +The old queue (Name compute) is set to default by slurm. +Each process over 60GB of RAM or over 20cpus should run on those new nodes. From 48400b8863b11dc2e70917fa3d586e56e5e41f6f Mon Sep 17 00:00:00 2001 From: mseybold Date: Fri, 6 Mar 2020 11:14:27 +0100 Subject: [PATCH 09/87] Update RADME.md fixed link to cfc.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0e58b09..3b360a1 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Currently documentation is available for the following systems: * [CCGA](docs/ccga.md) * [CCGA_DX](docs/ccga_dx.md) * [CCGA_MED](docs/ccga_med.md) -* [CFC](docs/binac.md) +* [CFC](docs/cfc.md) * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) From f5364e45e7c007d9da5b24dcb50c5329b53b36ae Mon Sep 17 00:00:00 2001 From: Evan Floden Date: Fri, 6 Mar 2020 11:27:44 +0100 Subject: [PATCH 10/87] Update google.md or markdown linting --- docs/google.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/google.md b/docs/google.md index d3a65f5..0534a0b 100644 --- a/docs/google.md +++ b/docs/google.md @@ -2,33 +2,36 @@ To be used with the `google` profile by specifying the `-profile google` when running nf-core pipelines. - ![Google Cloud](images/google-cloud-logo.svg "https://cloud.google.com/life-sciences/docs/tutorials/nextflow") - ## Quick Start -``` + +```bash GOOGLE_APPLICATION_CREDENTIALS=.json NXF_MODE=google nextflow run nf-core/rnaseq -profile google,test --google_bucket ``` ### Required Parameters + #### `--google-bucket` + The Google Cloud Storage bucket location to be used as a Nextflow work directory. Can also be specified with (`-w gs://your_bucket/work`). ### Optional Parameters + #### `--google-zone` + The Google zone where the computation is executed in Compute Engine VMs. Multiple zones can be provided separating them by a comma. Default (`europe-west2-c`). #### `--google-preemptible` + Enables the usage of preemptible virtual machines with a retry error statergy for up to 5 retries. Default (`true`). #### `--google-debug` + Copies the /google debug directory from the VM to the task bucket directory. Useful for debugging. Default (`false`). - ## Cloud Life Sciences Setup + Please refer to the [Google Cloud](https://cloud.google.com/life-sciences/docs/tutorials/nextflow) and [Nextflow](https://www.nextflow.io/docs/latest/google.html#cloud-life-sciences) documentation which describe how to setup the Google Cloud environment. - - From ebae81ceb3a5a9d07d72d31b33867a369944d889 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 6 Mar 2020 17:18:03 +0100 Subject: [PATCH 11/87] Update main.yml Drop CCGA, as it was removed in an intermediate merge. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5e9676d..36f9da1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 6141e008a3f082bb048aa5f41743bcc4bc706270 Mon Sep 17 00:00:00 2001 From: Evan Floden Date: Mon, 9 Mar 2020 14:44:58 +0100 Subject: [PATCH 12/87] Update google.md --- docs/google.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/google.md b/docs/google.md index 0534a0b..9e8c521 100644 --- a/docs/google.md +++ b/docs/google.md @@ -9,7 +9,7 @@ To be used with the `google` profile by specifying the `-profile google` when ru ```bash GOOGLE_APPLICATION_CREDENTIALS=.json NXF_MODE=google -nextflow run nf-core/rnaseq -profile google,test --google_bucket +nextflow run nf-core/rnaseq -profile test,google --google_bucket ``` ### Required Parameters From de914e56eed0a86d739f3f5fe74897858252aa69 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:08:01 +0200 Subject: [PATCH 13/87] Create ebc.config --- conf/ebc.config | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 conf/ebc.config diff --git a/conf/ebc.config b/conf/ebc.config new file mode 100644 index 0000000..ebcb775 --- /dev/null +++ b/conf/ebc.config @@ -0,0 +1,26 @@ + //Profile config names for nf-core/configs + params { + config_profile_description = 'Generic Estonian Biocentre profile provided by nf-core/configs.' + config_profile_contact = 'Marcel Keller (@marcel-keller)' + config_profile_url = 'https://genomics.ut.ee/en/about-us/estonian-biocentre' + } + + cleanup = true + + conda { + cacheDir = '/ebc_data/nf-core/conda' + + } + process { + executor = 'slurm' + conda = "$baseDir/environment.yml" + + } + executor { + queueSize = 16 + } + params { + max_memory = 12.GB + max_cpus = 20 + max_time = 120.h + } From b8d020b78c54b162f5178f531fcc4dad9c69f5b5 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 9 Mar 2020 15:12:49 +0100 Subject: [PATCH 14/87] Update cfc.md --- docs/cfc.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/cfc.md b/docs/cfc.md index acf93dd..f96a47d 100644 --- a/docs/cfc.md +++ b/docs/cfc.md @@ -15,6 +15,5 @@ module load devel/singularity/3.4.2 >NB: You will need an account to use the HPC cluster CFC in order to run the pipeline. If in doubt contact IT. >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. -New HighMem nodes are available on the cfc, Partition/queue name is qbic, each node has 128Cores and 2TB of RAM. -The old queue (Name compute) is set to default by slurm. -Each process over 60GB of RAM or over 20cpus should run on those new nodes. + +The queues are set to be `qbic` or `compute` and will be chosen automatically for you depending on your job submission. From 418f4b52b66125566aff9e985bc6ef9b30229875 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:23:16 +0200 Subject: [PATCH 15/87] Create ebc.md --- docs/ebc.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/ebc.md diff --git a/docs/ebc.md b/docs/ebc.md new file mode 100644 index 0000000..f997f9a --- /dev/null +++ b/docs/ebc.md @@ -0,0 +1,8 @@ +# nf-core/configs: EBC Configuration +All nf-core pipelines have been successfully configured for use on the [Estonian Biocentre (EBC)](https://genomics.ut.ee/en/about-us/estonian-biocentre) cluster at the [High Performance Computing Center](https://hpc.ut.ee/en) of the the University of Tartu. +To use, run the pipeline with `-profile ebc`. This will download and launch the [`ebc.config`](../conf/ebc.config) which has been pre-configured with a setup suitable for the EBC cluster. Using this profile, currently, a conda environment containing all of the required software will be downloaded and stored in a central location. + +The profile will put a maximum job limit of 12 GB, 20 CPUs and a maximum wall time of 120 hours. + +NB: You will need an account to use the HPC cluster on EBC cluster in order to run the pipeline. If in doubt contact IT. +NB: Nextflow will need to submit the jobs via the SLURM scheduler to the HPC cluster and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. From f48a4cbfb8f4a3ad8845d2db7bf13759bb4c5888 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:25:19 +0200 Subject: [PATCH 16/87] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 36f9da1..042dd26 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 88cfe1118738067c8d47ecb567e5d4e0b0184cc1 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:32:14 +0200 Subject: [PATCH 17/87] Update ebc.md --- docs/ebc.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ebc.md b/docs/ebc.md index f997f9a..45c9415 100644 --- a/docs/ebc.md +++ b/docs/ebc.md @@ -1,4 +1,5 @@ # nf-core/configs: EBC Configuration + All nf-core pipelines have been successfully configured for use on the [Estonian Biocentre (EBC)](https://genomics.ut.ee/en/about-us/estonian-biocentre) cluster at the [High Performance Computing Center](https://hpc.ut.ee/en) of the the University of Tartu. To use, run the pipeline with `-profile ebc`. This will download and launch the [`ebc.config`](../conf/ebc.config) which has been pre-configured with a setup suitable for the EBC cluster. Using this profile, currently, a conda environment containing all of the required software will be downloaded and stored in a central location. From edc6a6aa87946ba943141298a7b262eb74aed754 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:35:45 +0200 Subject: [PATCH 18/87] Update nfcore_custom.config --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index d7f3ca4..848db80 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -22,6 +22,7 @@ profiles { czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config"} + ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" } google { includeConfig "${params.custom_config_base}/conf/google.config" } denbi_qbic { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" } From 3b575195009b748cf632b4cd5294d32e3edc031a Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:48:50 +0200 Subject: [PATCH 19/87] Update ebc.config --- conf/ebc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/ebc.config b/conf/ebc.config index ebcb775..4b1332d 100644 --- a/conf/ebc.config +++ b/conf/ebc.config @@ -14,7 +14,7 @@ process { executor = 'slurm' conda = "$baseDir/environment.yml" - + beforeScript = 'module load nextflow' } executor { queueSize = 16 From c00bbe982b7250898c9310e8bb9fbb6e774f1e13 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:51:38 +0200 Subject: [PATCH 20/87] Update ebc.config --- conf/ebc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/ebc.config b/conf/ebc.config index 4b1332d..ea48f05 100644 --- a/conf/ebc.config +++ b/conf/ebc.config @@ -9,12 +9,12 @@ conda { cacheDir = '/ebc_data/nf-core/conda' - } process { executor = 'slurm' conda = "$baseDir/environment.yml" beforeScript = 'module load nextflow' + queue = { task.time <= 24.h ? 'testing' : 'main' } } executor { queueSize = 16 From 8e3803e4d7dc8b95f15a084f4e9790bd8731fe5b Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:57:45 +0200 Subject: [PATCH 21/87] Update ebc.config --- conf/ebc.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/ebc.config b/conf/ebc.config index ea48f05..4df0596 100644 --- a/conf/ebc.config +++ b/conf/ebc.config @@ -14,7 +14,6 @@ executor = 'slurm' conda = "$baseDir/environment.yml" beforeScript = 'module load nextflow' - queue = { task.time <= 24.h ? 'testing' : 'main' } } executor { queueSize = 16 From dcb478c904d35b3b49db20abc4a2ecdf990e3dc1 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 17:07:03 +0200 Subject: [PATCH 22/87] Update ebc.md --- docs/ebc.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ebc.md b/docs/ebc.md index 45c9415..71f2622 100644 --- a/docs/ebc.md +++ b/docs/ebc.md @@ -3,6 +3,8 @@ All nf-core pipelines have been successfully configured for use on the [Estonian Biocentre (EBC)](https://genomics.ut.ee/en/about-us/estonian-biocentre) cluster at the [High Performance Computing Center](https://hpc.ut.ee/en) of the the University of Tartu. To use, run the pipeline with `-profile ebc`. This will download and launch the [`ebc.config`](../conf/ebc.config) which has been pre-configured with a setup suitable for the EBC cluster. Using this profile, currently, a conda environment containing all of the required software will be downloaded and stored in a central location. +> :warning: You must install your own [conda binary](conda.io) to run nf-core pipelines in a conda environment. Running with singularity will be added soon. + The profile will put a maximum job limit of 12 GB, 20 CPUs and a maximum wall time of 120 hours. NB: You will need an account to use the HPC cluster on EBC cluster in order to run the pipeline. If in doubt contact IT. From 123c44c56daf8fa6f06ed46a2f6ca454ddc653ca Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 9 Mar 2020 16:18:41 -0500 Subject: [PATCH 23/87] feat: add initial ganymede config --- conf/utd_ganymede.config | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 conf/utd_ganymede.config diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config new file mode 100644 index 0000000..a183861 --- /dev/null +++ b/conf/utd_ganymede.config @@ -0,0 +1,24 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'University of Texas at Dallas HPC cluster profile provided by nf-core/configs' + config_profile_contact = 'Edmund Miller(@emiller88)' + config_profile_url = 'http://docs.oithpc.utdallas.edu/' +} + +singularity { + enabled = true + envWhitelist='SINGULARITY_BINDPATH' + autoMounts = true +} + +process { + beforeScript = 'module load Singularity/2.4.5' + executor = 'slurm' + queue = 'genomics' +} + +params { + max_memory = 32.GB + max_cpus = 16 + max_time = 48.h +} From 1c623953322329c05345517940a5d19fbf678fb9 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 9 Mar 2020 20:07:46 -0500 Subject: [PATCH 24/87] docs(utd_ganymede): add initial documentation --- README.md | 1 + docs/utd_ganymede.md | 18 ++++++++++++++++++ nfcore_custom.config | 4 +++- 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 docs/utd_ganymede.md diff --git a/README.md b/README.md index 755b1d2..686b157 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,7 @@ Currently documentation is available for the following systems: * [SHH](docs/shh.md) * [UCT_HEX](docs/uct_hex.md) * [UPPMAX](docs/uppmax.md) +* [UTD_GANYMEDE](docs/utd_ganymede.md) * [UZH](docs/uzh.md) ### Uploading to `nf-core/configs` diff --git a/docs/utd_ganymede.md b/docs/utd_ganymede.md new file mode 100644 index 0000000..81dd46a --- /dev/null +++ b/docs/utd_ganymede.md @@ -0,0 +1,18 @@ +# nf-core/configs: UTD Ganymede Configuration + +All nf-core pipelines have been successfully configured for use on the Ganymede HPC cluster at the [The Univeristy of Texas at Dallas](https://www.utdallas.edu/). + +To use, run the pipeline with `-profile utd_ganymede`. This will download and launch the [`utd_ganymede.config`](../conf/utd_ganymede.config) which has been pre-configured with a setup suitable for the Ganymede HPC cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to load Singularity using the environment module system on Ganymede. You can do this by issuing the commands below: + +```bash +## Singularity environment modules +module purge +module load singularity +``` + +All of the intermediate files required to run the pipeline will be stored in the `work/` directory. It is recommended to delete this directory after the pipeline has finished successfully because it can get quite large, and all of the main output files will be saved in the `results/` directory anyway. + +>NB: You will need an account to use the HPC cluster on Ganymede in order to run the pipeline. If in doubt contact Ganymedeadmins. +>NB: Nextflow will need to submit the jobs via SLURM to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact GanymedeAdmins. diff --git a/nfcore_custom.config b/nfcore_custom.config index 848db80..383eaf0 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -37,6 +37,7 @@ profiles { shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } + utd_ganymede { includeConfig "${params.custom_config_base}/conf/utd_ganymede.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } @@ -50,6 +51,7 @@ params { crick: ['.thecrick.org'], genotoul: ['.genologin1.toulouse.inra.fr', '.genologin2.toulouse.inra.fr'], genouest: ['.genouest.org'], - uppmax: ['.uppmax.uu.se'] + uppmax: ['.uppmax.uu.se'], + utd_ganymede: ['ganymede.utdallas.edu'] ] } From b79f0f0a21f017e21d07bba4816620a41063c31d Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 9 Mar 2020 20:11:52 -0500 Subject: [PATCH 25/87] ci: add utd_gaymede to main --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 042dd26..0917dd2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From e395007ba8a916ec0abc96b5bcfd500ed65431b1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 10 Mar 2020 09:34:30 +0100 Subject: [PATCH 26/87] Add reference to EBC in README docs --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 755b1d2..3d796ee 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ Currently documentation is available for the following systems: * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) +* [EBC](docs/ebc.md) * [DENBI_QBIC](docs/denbi_qbic.md) * [GENOTOUL](docs/genotoul.md) * [GENOUEST](docs/genouest.md) From 55c17a185451e6b6221e36418c7d2bbc88af15f4 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Tue, 10 Mar 2020 11:09:30 +0200 Subject: [PATCH 27/87] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 755b1d2..a4be2a9 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ Currently documentation is available for the following systems: * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) * [DENBI_QBIC](docs/denbi_qbic.md) +* [EBC](docs/ebc.md) * [GENOTOUL](docs/genotoul.md) * [GENOUEST](docs/genouest.md) * [GIS](docs/gis.md) From c42bb079817a84f87144fcbe6c386419a327ff4c Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 10 Mar 2020 09:55:17 -0500 Subject: [PATCH 28/87] fix: Singularity => singularity For some reason ours are lowercase --- conf/utd_ganymede.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index a183861..8b72448 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -12,7 +12,7 @@ singularity { } process { - beforeScript = 'module load Singularity/2.4.5' + beforeScript = 'module load singularity' executor = 'slurm' queue = 'genomics' } From cece8536636aade75dc2b38fe0cede689726dedc Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 10 Mar 2020 21:03:24 -0500 Subject: [PATCH 29/87] fix(ganymede): use singularity/2.4.5 Our 3.2.1 doesn't work right with snakemake or nextflow --- conf/utd_ganymede.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index 8b72448..465deb5 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -12,7 +12,7 @@ singularity { } process { - beforeScript = 'module load singularity' + beforeScript = 'module load singularity/2.4.5' executor = 'slurm' queue = 'genomics' } From dd1ddf864b100c416fd376905d6d602d7db9e76a Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 11 Mar 2020 14:01:28 +0100 Subject: [PATCH 30/87] update sarek munin config --- conf/pipeline/sarek/munin.config | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index e4413f2..ff31a7f 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -1,4 +1,4 @@ -// Profile config names for nf-core/configs + // sarek/munin specific profile config params { // Specific nf-core/configs params @@ -6,17 +6,23 @@ params { config_profile_description = 'nf-core/sarek MUNIN profile provided by nf-core/configs' // Specific nf-core/sarek params - annotation_cache = true - pon = '/data1/PON/vcfs/BTB.PON.vcf.gz' - pon_index = '/data1/PON/vcfs/BTB.PON.vcf.gz.tbi' - snpEff_cache = '/data1/cache/snpEff/' - vep_cache = '/data1/cache/VEP/' + annotation_cache = true + cadd_cache = true + cadd_indels = '/data1/cache/CADD/v1.4/InDels.tsv.gz' + cadd_indels_tbi = '/data1/cache/CADD/v1.4/InDels.tsv.gz.tbi' + cadd_wg_snvs = '/data1/cache/CADD/v1.4/whole_genome_SNVs.tsv.gz' + cadd_wg_snvs_tbi = '/data1/cache/CADD/v1.4/whole_genome_SNVs.tsv.gz.tbi' + pon = '/data1/PON/vcfs/BTB.PON.vcf.gz' + pon_index = '/data1/PON/vcfs/BTB.PON.vcf.gz.tbi' + snpeff_cache = '/data1/cache/snpEff/' + vep_cache = '/data1/cache/VEP/' + vep_cache_version = 95 } // Specific nf-core/sarek process configuration process { withLabel:sentieon { - module = {params.sentieon ? 'sentieon/201808.05' : null} + module = {params.sentieon ? 'sentieon/201911.00' : null} container = {params.sentieon ? null : container} } } From d24a296a016e437ec62babc81bd2004f35629be5 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 11 Mar 2020 14:02:01 +0100 Subject: [PATCH 31/87] add munin/rnafusion specific config --- conf/pipeline/rnafusion/munin.config | 64 ++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 conf/pipeline/rnafusion/munin.config diff --git a/conf/pipeline/rnafusion/munin.config b/conf/pipeline/rnafusion/munin.config new file mode 100644 index 0000000..d44a1f3 --- /dev/null +++ b/conf/pipeline/rnafusion/munin.config @@ -0,0 +1,64 @@ +/* + * ------------------------------------------------- + * Nextflow config file for Munin Singularity + * ------------------------------------------------- + */ + +params { + max_cpus = 24 + max_memory = 256.GB + max_time = 72.h + container_version = '1.1.0' + + // Tool versions + arriba_version = '1.1.0' + ericscript_version = '0.5.5' + fusioncatcher_version = '1.20' + fusion_inspector_version = '1.3.1-star2.7.0f' + pizzly_version = '0.37.3' + squid_version = '1.5-star2.7.0f' + star_fusion_version = '1.6.0' + + // Paths + reference_base = '/data1/references/rnafusion' + containerPath = "file:///data1/containers/rnafusion_containers_v${container_version}" + containerPathCommon = "file:///data1/containers/rnafusion_common" + + // References + fasta = "${params.reference_base}/1.1.0/Homo_sapiens.GRCh38_r97.all.fa" + gtf = "${params.reference_base}/1.1.0/Homo_sapiens.GRCh38_r97.gtf" + transcript = "${params.reference_base}/1.1.0/Homo_sapiens.GRCh38_r97.cdna.all.fa.gz" + databases = "${params.reference_base}/1.1.0/databases" + + star_index = "${params.reference_base}/1.1.0/star_index/star" + arriba_ref = "${params.reference_base}/arriba_ref" + fusioncatcher_ref = "${params.reference_base}/fusioncatcher_ref/human_v98" + star_fusion_ref = "${params.reference_base}/star_fusion/1.6.0/ensembl/ctat_genome_lib_build_dir" + ericscript_ref = "${params.reference_base}/ericscript_ref/ericscript_db_homosapiens_ensembl84" +} + +process { + container = "${params.containerPath}/rnafusion_1.1.0.img" + + withName: "arriba|arriba_visualization" { + container = "${params.containerPath}/rnafusion_arriba_v${params.arriba_version}.img" + } + withName: "star_fusion|download_star_fusion" { + container = "${params.containerPath}/rnafusion_star-fusion_v${params.star_fusion_version}.img" + } + withName:fusioncatcher { + container = "${params.containerPath}/rnafusion_fusioncatcher_v${params.fusioncatcher_version}.img" + } + withName:fusion_inspector { + container = "${params.containerPath}/rnafusion_fusion-inspector_v${params.fusion_inspector_version}.img" + } + withName:ericscript { + container = "${params.containerPathCommon}/rnafusion_ericscript_v${params.ericscript_version}.img" + } + withName:pizzly { + container = "${params.containerPathCommon}/rnafusion_pizzly_v${params.pizzly_version}.img" + } + withName:squid { + container = "${params.containerPath}/rnafusion_squid_v${params.squid_version}.img" + } +} \ No newline at end of file From 250035e71d0655c87f02a10749738505acd1c847 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 11 Mar 2020 14:11:23 +0100 Subject: [PATCH 32/87] fix path to config files --- docs/pipeline/ampliseq/binac.md | 2 +- docs/pipeline/eager/shh.md | 2 +- docs/pipeline/sarek/munin.md | 2 +- docs/pipeline/sarek/uppmax.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/pipeline/ampliseq/binac.md b/docs/pipeline/ampliseq/binac.md index a63e021..fb3403e 100644 --- a/docs/pipeline/ampliseq/binac.md +++ b/docs/pipeline/ampliseq/binac.md @@ -6,7 +6,7 @@ Extra specific configuration for the ampliseq pipeline. To use, run the pipeline with `-profile binac`. -This will download and launch the ampliseq specific [`binac.config`](../conf/pipeline/ampliseq/binac.config) which has been pre-configured with a setup suitable for the BINAC cluster. +This will download and launch the ampliseq specific [`binac.config`](../../../conf/pipeline/ampliseq/binac.config) which has been pre-configured with a setup suitable for the BINAC cluster. Example: `nextflow run nf-core/ampliseq -profile binac` diff --git a/docs/pipeline/eager/shh.md b/docs/pipeline/eager/shh.md index a0dc092..b77ede4 100644 --- a/docs/pipeline/eager/shh.md +++ b/docs/pipeline/eager/shh.md @@ -6,7 +6,7 @@ Extra specific configuration for eager pipeline To use, run the pipeline with `-profile shh`. -This will download and launch the eager specific [`shh.config`](../conf/pipeline/eager/shh.config) which has been pre-configured with a setup suitable for the shh cluster. +This will download and launch the eager specific [`shh.config`](../../../conf/pipeline/eager/shh.config) which has been pre-configured with a setup suitable for the shh cluster. Example: `nextflow run nf-core/eager -profile shh` diff --git a/docs/pipeline/sarek/munin.md b/docs/pipeline/sarek/munin.md index ce411c1..fc1636f 100644 --- a/docs/pipeline/sarek/munin.md +++ b/docs/pipeline/sarek/munin.md @@ -6,7 +6,7 @@ Extra specific configuration for sarek pipeline To use, run the pipeline with `-profile munin`. -This will download and launch the sarek specific [`munin.config`](../conf/pipeline/sarek/munin.config) which has been pre-configured with a setup suitable for the MUNIN cluster. +This will download and launch the sarek specific [`munin.config`](../../../conf/pipeline/sarek/munin.config) which has been pre-configured with a setup suitable for the MUNIN cluster. Example: `nextflow run nf-core/sarek -profile munin` diff --git a/docs/pipeline/sarek/uppmax.md b/docs/pipeline/sarek/uppmax.md index ab8d85e..e612a41 100644 --- a/docs/pipeline/sarek/uppmax.md +++ b/docs/pipeline/sarek/uppmax.md @@ -6,7 +6,7 @@ Extra specific configuration for sarek pipeline To use, run the pipeline with `-profile uppmax`. -This will download and launch the sarek specific [`uppmax.config`](../conf/pipeline/sarek/uppmax.config) which has been pre-configured with a setup suitable for uppmax clusters. +This will download and launch the sarek specific [`uppmax.config`](../../../conf/pipeline/sarek/uppmax.config) which has been pre-configured with a setup suitable for uppmax clusters. Example: `nextflow run nf-core/sarek -profile uppmax` From 7c60096952539b7266cb82200f0dee77b5cef742 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 11 Mar 2020 14:17:24 +0100 Subject: [PATCH 33/87] update docs for munin/sarek specific config --- docs/pipeline/sarek/munin.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/pipeline/sarek/munin.md b/docs/pipeline/sarek/munin.md index fc1636f..5f9511b 100644 --- a/docs/pipeline/sarek/munin.md +++ b/docs/pipeline/sarek/munin.md @@ -6,17 +6,22 @@ Extra specific configuration for sarek pipeline To use, run the pipeline with `-profile munin`. -This will download and launch the sarek specific [`munin.config`](../../../conf/pipeline/sarek/munin.config) which has been pre-configured with a setup suitable for the MUNIN cluster. +This will download and launch the sarek specific [`munin.config`](../../../conf/pipeline/sarek/munin.config) which has been pre-configured with a setup suitable for the `MUNIN` cluster. Example: `nextflow run nf-core/sarek -profile munin` ## Sarek specific configurations for MUNIN -Specific configurations for MUNIN has been made for sarek. +Specific configurations for `MUNIN` has been made for sarek. -* Params `annotation_cache` set to `true` -* Path to `snpEff_cache`: `/data1/cache/snpEff/` +* Params `annotation_cache` and `cadd_cache` set to `true` +* Params `vep_cache_version` set to `95` +* Path to `snpeff_cache`: `/data1/cache/snpEff/` * Path to `vep_cache`: `/data1/cache/VEP/` * Path to `pon`: `/data1/PON/vcfs/BTB.PON.vcf.gz` * Path to `pon_index`: `/data1/PON/vcfs/BTB.PON.vcf.gz.tbi` +* Path to `cadd_indels`: `/data1/cache/CADD/v1.4/InDels.tsv.gz` +* Path to `cadd_indels_tbi`: `/data1/cache/CADD/v1.4/InDels.tsv.gz.tbi` +* Path to `cadd_wg_snvs`: `/data1/cache/CADD/v1.4/whole_genome_SNVs.tsv.gz` +* Path to `cadd_wg_snvs_tbi`: `/data1/cache/CADD/v1.4/whole_genome_SNVs.tsv.gz.tbi` * Load module `Sentieon` for Processes with `sentieon` labels From d4bea56f5241d6dee2aa9ba9d7ec09cdbcf8210f Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 11 Mar 2020 14:23:23 +0100 Subject: [PATCH 34/87] add munin/rnafusion specific docs + markdownlint --- README.md | 104 ++++++++++++++++--------------- docs/pipeline/rnafusion/munin.md | 19 ++++++ 2 files changed, 72 insertions(+), 51 deletions(-) create mode 100644 docs/pipeline/rnafusion/munin.md diff --git a/README.md b/README.md index 94f999a..f38242d 100644 --- a/README.md +++ b/README.md @@ -6,19 +6,19 @@ A repository for hosting Nextflow configuration files containing custom paramete ## Table of contents -* [Using an existing config](#using-an-existing-config) - * [Configuration and parameters](#configuration-and-parameters) - * [Offline usage](#offline-usage) -* [Adding a new config](#adding-a-new-config) - * [Checking user hostnames](#checking-user-hostnames) - * [Testing](#testing) - * [Documentation](#documentation) - * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) -* [Adding a new pipeline-specific config](#adding-a-new-pipeline-specific-config) - * [Pipeline-specific documentation](#pipeline-specific-documentation) - * [Enabling pipeline-specific configs within a pipeline](#enabling-pipeline-specific-configs-within-a-pipeline) - * [Create the pipeline-specific `nf-core/configs` files](#create-the-pipeline-specific-nf-coreconfigs-files) -* [Help](#help) +- [Using an existing config](#using-an-existing-config) + - [Configuration and parameters](#configuration-and-parameters) + - [Offline usage](#offline-usage) +- [Adding a new config](#adding-a-new-config) + - [Checking user hostnames](#checking-user-hostnames) + - [Testing](#testing) + - [Documentation](#documentation) + - [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) +- [Adding a new pipeline-specific config](#adding-a-new-pipeline-specific-config) + - [Pipeline-specific documentation](#pipeline-specific-documentation) + - [Enabling pipeline-specific configs within a pipeline](#enabling-pipeline-specific-configs-within-a-pipeline) + - [Create the pipeline-specific `nf-core/configs` files](#create-the-pipeline-specific-nf-coreconfigs-files) +- [Help](#help) ## Using an existing config @@ -93,33 +93,33 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs Currently documentation is available for the following systems: -* [AWSBATCH](docs/awsbatch.md) -* [BIGPURPLE](docs/bigpurple.md) -* [BINAC](docs/binac.md) -* [CBE](docs/cbe.md) -* [CCGA_DX](docs/ccga_dx.md) -* [CCGA_MED](docs/ccga_med.md) -* [CFC](docs/cfc.md) -* [CRICK](docs/crick.md) -* [CZBIOHUB_AWS](docs/czbiohub.md) -* [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) -* [DENBI_QBIC](docs/denbi_qbic.md) -* [EBC](docs/ebc.md) -* [GENOTOUL](docs/genotoul.md) -* [GENOUEST](docs/genouest.md) -* [GIS](docs/gis.md) -* [GOOGLE](docs/google.md) -* [HEBBE](docs/hebbe.md) -* [KRAKEN](docs/kraken.md) -* [MUNIN](docs/munin.md) -* [PASTEUR](docs/pasteur.md) -* [PHOENIX](docs/phoenix.md) -* [PRINCE](docs/prince.md) -* [SHH](docs/shh.md) -* [UCT_HEX](docs/uct_hex.md) -* [UPPMAX](docs/uppmax.md) -* [UTD_GANYMEDE](docs/utd_ganymede.md) -* [UZH](docs/uzh.md) +- [AWSBATCH](docs/awsbatch.md) +- [BIGPURPLE](docs/bigpurple.md) +- [BINAC](docs/binac.md) +- [CBE](docs/cbe.md) +- [CCGA_DX](docs/ccga_dx.md) +- [CCGA_MED](docs/ccga_med.md) +- [CFC](docs/cfc.md) +- [CRICK](docs/crick.md) +- [CZBIOHUB_AWS](docs/czbiohub.md) +- [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) +- [DENBI_QBIC](docs/denbi_qbic.md) +- [EBC](docs/ebc.md) +- [GENOTOUL](docs/genotoul.md) +- [GENOUEST](docs/genouest.md) +- [GIS](docs/gis.md) +- [GOOGLE](docs/google.md) +- [HEBBE](docs/hebbe.md) +- [KRAKEN](docs/kraken.md) +- [MUNIN](docs/munin.md) +- [PASTEUR](docs/pasteur.md) +- [PHOENIX](docs/phoenix.md) +- [PRINCE](docs/prince.md) +- [SHH](docs/shh.md) +- [UCT_HEX](docs/uct_hex.md) +- [UPPMAX](docs/uppmax.md) +- [UTD_GANYMEDE](docs/utd_ganymede.md) +- [UZH](docs/uzh.md) ### Uploading to `nf-core/configs` @@ -163,13 +163,15 @@ Note that pipeline-specific configs are not required and should only be added if Currently documentation is available for the following pipeline within the specific profile: -* ampliseq - * [BINAC](docs/pipeline/ampliseq/binac.md) -* eager - * [SHH](docs/pipeline/eager/shh.md) -* sarek - * [MUNIN](docs/pipeline/sarek/munin.md) - * [UPPMAX](docs/pipeline/sarek/uppmax.md) +- ampliseq + - [BINAC](docs/pipeline/ampliseq/binac.md) +- eager + - [SHH](docs/pipeline/eager/shh.md) +- rnafusion + - [MUNIN](docs/pipeline/rnafusion/munin.md) +- sarek + - [MUNIN](docs/pipeline/sarek/munin.md) + - [UPPMAX](docs/pipeline/sarek/uppmax.md) ### Enabling pipeline-specific configs within a pipeline @@ -198,7 +200,7 @@ We will be notified automatically when you have created your pull request, and p [Fork](https://help.github.com/articles/fork-a-repo/) the [`nf-core/configs`](https://github.com/nf-core/configs/) repository to your own GitHub account. And add or edit the following files in the local clone of your fork. -* `pipeline/.config` +- `pipeline/.config` If not already created, create the `pipeline/.config` file, and add your custom profile to the profile scope @@ -208,18 +210,18 @@ profiles { } ``` -* `conf/pipeline//.config` +- `conf/pipeline//.config` Add the custom configuration file to the `conf/pipeline//` directory. Make sure to add an extra `params` section with `params.config_profile_description`, `params.config_profile_contact` to the top of `pipeline/.config` and set to reasonable values. Users will get information on who wrote the pipeline-specific configuration profile then when executing the nf-core pipeline and can report back if there are things missing for example. -* `docs/pipeline//.md` +- `docs/pipeline//.md` Add the documentation file to the `docs/pipeline//` directory. You will also need to edit and add your custom profile to the [`README.md`](https://github.com/nf-core/configs/blob/master/README.md) file in the top-level directory of the clone. -* `README.md` +- `README.md` Edit this file, and add the new pipeline-specific institutional profile to the list in the section Pipeline specific documentation diff --git a/docs/pipeline/rnafusion/munin.md b/docs/pipeline/rnafusion/munin.md new file mode 100644 index 0000000..daa1dd1 --- /dev/null +++ b/docs/pipeline/rnafusion/munin.md @@ -0,0 +1,19 @@ +# nf-core/configs: MUNIN rnafusion specific configuration + +Extra specific configuration for rnafusion pipeline + +## Usage + +To use, run the pipeline with `-profile munin`. + +This will download and launch the rnafusion specific [`munin.config`](../../../conf/pipeline/rnafusion/munin.config) which has been pre-configured with a setup suitable for the `MUNIN` cluster. + +Example: `nextflow run nf-core/rnafusion -profile munin` + +## rnafusion specific configurations for MUNIN + +Specific configurations for `MUNIN` has been made for rnafusion. + +* `cpus`, `memory` and `time` max requirements. +* Paths to specific containers +* Paths to specific references and indexes From 6be17167972fff5616c29c485ecb1ed0a0e50acb Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 11 Mar 2020 15:39:36 +0100 Subject: [PATCH 35/87] revert * to - changes --- README.md | 80 +++++++++++++++++++++++++++---------------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index f38242d..3b60196 100644 --- a/README.md +++ b/README.md @@ -93,33 +93,33 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs Currently documentation is available for the following systems: -- [AWSBATCH](docs/awsbatch.md) -- [BIGPURPLE](docs/bigpurple.md) -- [BINAC](docs/binac.md) -- [CBE](docs/cbe.md) -- [CCGA_DX](docs/ccga_dx.md) -- [CCGA_MED](docs/ccga_med.md) -- [CFC](docs/cfc.md) -- [CRICK](docs/crick.md) -- [CZBIOHUB_AWS](docs/czbiohub.md) -- [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) -- [DENBI_QBIC](docs/denbi_qbic.md) -- [EBC](docs/ebc.md) -- [GENOTOUL](docs/genotoul.md) -- [GENOUEST](docs/genouest.md) -- [GIS](docs/gis.md) -- [GOOGLE](docs/google.md) -- [HEBBE](docs/hebbe.md) -- [KRAKEN](docs/kraken.md) -- [MUNIN](docs/munin.md) -- [PASTEUR](docs/pasteur.md) -- [PHOENIX](docs/phoenix.md) -- [PRINCE](docs/prince.md) -- [SHH](docs/shh.md) -- [UCT_HEX](docs/uct_hex.md) -- [UPPMAX](docs/uppmax.md) -- [UTD_GANYMEDE](docs/utd_ganymede.md) -- [UZH](docs/uzh.md) +* [AWSBATCH](docs/awsbatch.md) +* [BIGPURPLE](docs/bigpurple.md) +* [BINAC](docs/binac.md) +* [CBE](docs/cbe.md) +* [CCGA_DX](docs/ccga_dx.md) +* [CCGA_MED](docs/ccga_med.md) +* [CFC](docs/cfc.md) +* [CRICK](docs/crick.md) +* [CZBIOHUB_AWS](docs/czbiohub.md) +* [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) +* [DENBI_QBIC](docs/denbi_qbic.md) +* [EBC](docs/ebc.md) +* [GENOTOUL](docs/genotoul.md) +* [GENOUEST](docs/genouest.md) +* [GIS](docs/gis.md) +* [GOOGLE](docs/google.md) +* [HEBBE](docs/hebbe.md) +* [KRAKEN](docs/kraken.md) +* [MUNIN](docs/munin.md) +* [PASTEUR](docs/pasteur.md) +* [PHOENIX](docs/phoenix.md) +* [PRINCE](docs/prince.md) +* [SHH](docs/shh.md) +* [UCT_HEX](docs/uct_hex.md) +* [UPPMAX](docs/uppmax.md) +* [UTD_GANYMEDE](docs/utd_ganymede.md) +* [UZH](docs/uzh.md) ### Uploading to `nf-core/configs` @@ -163,15 +163,15 @@ Note that pipeline-specific configs are not required and should only be added if Currently documentation is available for the following pipeline within the specific profile: -- ampliseq - - [BINAC](docs/pipeline/ampliseq/binac.md) -- eager - - [SHH](docs/pipeline/eager/shh.md) -- rnafusion - - [MUNIN](docs/pipeline/rnafusion/munin.md) -- sarek - - [MUNIN](docs/pipeline/sarek/munin.md) - - [UPPMAX](docs/pipeline/sarek/uppmax.md) +* ampliseq + * [BINAC](docs/pipeline/ampliseq/binac.md) +* eager + * [SHH](docs/pipeline/eager/shh.md) +* rnafusion + * [MUNIN](docs/pipeline/rnafusion/munin.md) +* sarek + * [MUNIN](docs/pipeline/sarek/munin.md) + * [UPPMAX](docs/pipeline/sarek/uppmax.md) ### Enabling pipeline-specific configs within a pipeline @@ -200,7 +200,7 @@ We will be notified automatically when you have created your pull request, and p [Fork](https://help.github.com/articles/fork-a-repo/) the [`nf-core/configs`](https://github.com/nf-core/configs/) repository to your own GitHub account. And add or edit the following files in the local clone of your fork. -- `pipeline/.config` +* `pipeline/.config` If not already created, create the `pipeline/.config` file, and add your custom profile to the profile scope @@ -210,18 +210,18 @@ profiles { } ``` -- `conf/pipeline//.config` +* `conf/pipeline//.config` Add the custom configuration file to the `conf/pipeline//` directory. Make sure to add an extra `params` section with `params.config_profile_description`, `params.config_profile_contact` to the top of `pipeline/.config` and set to reasonable values. Users will get information on who wrote the pipeline-specific configuration profile then when executing the nf-core pipeline and can report back if there are things missing for example. -- `docs/pipeline//.md` +* `docs/pipeline//.md` Add the documentation file to the `docs/pipeline//` directory. You will also need to edit and add your custom profile to the [`README.md`](https://github.com/nf-core/configs/blob/master/README.md) file in the top-level directory of the clone. -- `README.md` +* `README.md` Edit this file, and add the new pipeline-specific institutional profile to the list in the section Pipeline specific documentation From 4a7f98a83b9de32869efbe993262afd2c88d9573 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 11 Mar 2020 15:41:58 +0100 Subject: [PATCH 36/87] update TOC --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 3b60196..95c1d79 100644 --- a/README.md +++ b/README.md @@ -6,19 +6,19 @@ A repository for hosting Nextflow configuration files containing custom paramete ## Table of contents -- [Using an existing config](#using-an-existing-config) - - [Configuration and parameters](#configuration-and-parameters) - - [Offline usage](#offline-usage) -- [Adding a new config](#adding-a-new-config) - - [Checking user hostnames](#checking-user-hostnames) - - [Testing](#testing) - - [Documentation](#documentation) - - [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) -- [Adding a new pipeline-specific config](#adding-a-new-pipeline-specific-config) - - [Pipeline-specific documentation](#pipeline-specific-documentation) - - [Enabling pipeline-specific configs within a pipeline](#enabling-pipeline-specific-configs-within-a-pipeline) - - [Create the pipeline-specific `nf-core/configs` files](#create-the-pipeline-specific-nf-coreconfigs-files) -- [Help](#help) +* [Using an existing config](#using-an-existing-config) + * [Configuration and parameters](#configuration-and-parameters) + * [Offline usage](#offline-usage) +* [Adding a new config](#adding-a-new-config) + * [Checking user hostnames](#checking-user-hostnames) + * [Testing](#testing) + * [Documentation](#documentation) + * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) +* [Adding a new pipeline-specific config](#adding-a-new-pipeline-specific-config) + * [Pipeline-specific documentation](#pipeline-specific-documentation) + * [Enabling pipeline-specific configs within a pipeline](#enabling-pipeline-specific-configs-within-a-pipeline) + * [Create the pipeline-specific `nf-core/configs` files](#create-the-pipeline-specific-nf-coreconfigs-files) +* [Help](#help) ## Using an existing config From 0f34d757a9eb061c6e1d075a769d9f83dea8f5dd Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 11 Mar 2020 15:48:17 +0100 Subject: [PATCH 37/87] Update munin.config --- conf/pipeline/rnafusion/munin.config | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/conf/pipeline/rnafusion/munin.config b/conf/pipeline/rnafusion/munin.config index d44a1f3..549d957 100644 --- a/conf/pipeline/rnafusion/munin.config +++ b/conf/pipeline/rnafusion/munin.config @@ -1,8 +1,4 @@ -/* - * ------------------------------------------------- - * Nextflow config file for Munin Singularity - * ------------------------------------------------- - */ +// rnafusion/munin specific profile config params { max_cpus = 24 @@ -61,4 +57,4 @@ process { withName:squid { container = "${params.containerPath}/rnafusion_squid_v${params.squid_version}.img" } -} \ No newline at end of file +} From b6d52fdb89665f022971e75fc4c0856275291ad3 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 11 Mar 2020 15:48:29 +0100 Subject: [PATCH 38/87] Update conf/pipeline/sarek/munin.config --- conf/pipeline/sarek/munin.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index ff31a7f..d22e5e9 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -1,4 +1,4 @@ - // sarek/munin specific profile config +// sarek/munin specific profile config params { // Specific nf-core/configs params From 123132d89a46ac9aec95373ef8a049204be4ea5f Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Thu, 12 Mar 2020 17:23:55 +0100 Subject: [PATCH 39/87] Update munin.config fix typo --- conf/pipeline/sarek/munin.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index d22e5e9..039d117 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -16,7 +16,7 @@ params { pon_index = '/data1/PON/vcfs/BTB.PON.vcf.gz.tbi' snpeff_cache = '/data1/cache/snpEff/' vep_cache = '/data1/cache/VEP/' - vep_cache_version = 95 + vep_cache_version = '95' } // Specific nf-core/sarek process configuration From 52369ba6adde0edff9f77e701e7119022ba943eb Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Tue, 17 Mar 2020 16:45:20 -0700 Subject: [PATCH 40/87] Add AaegL5.0 config --- conf/czbiohub_aws.config | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index 9d17a48..b32fb6d 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -50,6 +50,7 @@ params { // No final slash because it's added later gencode_base = "s3://czbiohub-reference/gencode" transgenes_base = "s3://czbiohub-reference/transgenes" + refseq_base = "s3://czbiohub-reference/ncbi/genomes/refseq/" // AWS configurations awsregion = "us-west-2" @@ -79,6 +80,12 @@ params { transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" star = "${params.gencode_base}/mouse/vM21/STARIndex/" } + 'AaegL5.0' { + fasta = "${params.refseq_base}/invertebrate/Aedes_aegypti/GCF_002204515.2_AaegL5.0/nf-core--rnaseq/reference_genome/GCF_002204515.2_AaegL5.0_genomic.fna" + gtf = "${params.refseq_base}/invertebrate/Aedes_aegypti/GCF_002204515.2_AaegL5.0/nf-core--rnaseq/reference_genome/GCF_002204515.2_AaegL5.0_genomic.gtf" + bed = "${params.refseq_base}/invertebrate/Aedes_aegypti/GCF_002204515.2_AaegL5.0/nf-core--rnaseq/reference_genome/GCF_002204515.2_AaegL5.0_genomic.bed" + star = "${params.refseq_base}/invertebrate/Aedes_aegypti/GCF_002204515.2_AaegL5.0/nf-core--rnaseq/reference_genome/star/" + } } transgenes { From c81f693f738da41002c7cf4db105e3391c89a725 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Mon, 23 Mar 2020 17:16:34 +0100 Subject: [PATCH 41/87] add cfc_dev config --- conf/cfc.config | 2 +- conf/cfc_dev.config | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 conf/cfc_dev.config diff --git a/conf/cfc.config b/conf/cfc.config index d1a109a..62d3626 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -1,7 +1,7 @@ //Profile config names for nf-core/configs params { config_profile_description = 'QBiC Core Facility cluster profile provided by nf-core/configs.' - config_profile_contact = 'Alexander Peltzer (@apeltzer)' + config_profile_contact = 'Gisela Gabernet (@ggabernet)' config_profile_url = 'http://qbic.uni-tuebingen.de/' } diff --git a/conf/cfc_dev.config b/conf/cfc_dev.config new file mode 100644 index 0000000..a7ebf2f --- /dev/null +++ b/conf/cfc_dev.config @@ -0,0 +1,28 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'QBiC Core Facility cluster dev profile without container cache provided by nf-core/configs.' + config_profile_contact = 'Gisela Gabernet (@ggabernet)' + config_profile_url = 'http://qbic.uni-tuebingen.de/' +} + +singularity { + enabled = true +} + +process { + beforeScript = 'module load devel/singularity/3.4.2' + executor = 'slurm' + queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' : 'compute' } +} + +weblog{ + enabled = true + url = 'https://services.qbic.uni-tuebingen.de/flowstore/workflows' +} + +params { + igenomes_base = '/nfsmounts/igenomes' + max_memory = 1999.GB + max_cpus = 128 + max_time = 140.h +} \ No newline at end of file From d02e2b8a36d362d7a55df4636d4cdba382536338 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Mon, 23 Mar 2020 17:22:02 +0100 Subject: [PATCH 42/87] add cfc_dev and warning for binac and cfc --- nfcore_custom.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 383eaf0..ea2238a 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -17,6 +17,7 @@ profiles { ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } ccga_med { includeConfig "${params.custom_config_base}/conf/ccga_med.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } + cfc_dev { includeConfig "${params.custom_config_base}/conf/cfc_dev.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } czbiohub_aws_highpriority { @@ -47,7 +48,9 @@ profiles { params { // This is a groovy map, not a nextflow parameter set hostnames = [ + binac: ['.binac.uni-tuebingen.de'], cbe: ['.cbe.vbc.ac.at'], + cfc: ['.hpc.uni-tuebingen.de'], crick: ['.thecrick.org'], genotoul: ['.genologin1.toulouse.inra.fr', '.genologin2.toulouse.inra.fr'], genouest: ['.genouest.org'], From fa01e6c5c18861f22b153619981ab4ecf8421dbf Mon Sep 17 00:00:00 2001 From: ggabernet Date: Mon, 23 Mar 2020 17:27:36 +0100 Subject: [PATCH 43/87] add cfc_dev to test yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0917dd2..bdc485b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 8b570507cd65b4e6ac7ccc3cf23adc9f26d080e1 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 30 Mar 2020 16:40:11 +0200 Subject: [PATCH 44/87] update s --- conf/pipeline/sarek/uppmax.config | 2 +- conf/uppmax.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/sarek/uppmax.config b/conf/pipeline/sarek/uppmax.config index 1e467dd..3feb2df 100644 --- a/conf/pipeline/sarek/uppmax.config +++ b/conf/pipeline/sarek/uppmax.config @@ -9,7 +9,7 @@ params { igenomeIgnore = true genomes_base = params.genome == 'GRCh37' ? '/sw/data/uppnex/ToolBox/ReferenceAssemblies/hg38make/bundle/2.8/b37' : '/sw/data/uppnex/ToolBox/hg38bundle' } - if (hostname ==~ "r.*") { +if (hostname ==~ "r.*") { params.singleCPUmem = 6400.MB } if (hostname ==~ "i.*") { diff --git a/conf/uppmax.config b/conf/uppmax.config index 9eb45fc..037070a 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -26,7 +26,7 @@ params { def hostname = "hostname".execute().text.trim() -if (hostname ==~ "b.*") { +if (hostname ==~ "b.*" || hostname ==~ "s.*") { params.max_memory = 109.GB } From c8de9be7014ee85359caddc04aefd725974d5d6b Mon Sep 17 00:00:00 2001 From: Daniel Lundin Date: Tue, 31 Mar 2020 15:17:24 +0200 Subject: [PATCH 45/87] Adding UPPMAX specific config for ampliseq --- conf/pipeline/ampliseq/uppmax.config | 15 +++++++++++++++ docs/pipeline/ampliseq/uppmax.md | 17 +++++++++++++++++ pipeline/ampliseq.config | 6 +++++- 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 conf/pipeline/ampliseq/uppmax.config create mode 100644 docs/pipeline/ampliseq/uppmax.md diff --git a/conf/pipeline/ampliseq/uppmax.config b/conf/pipeline/ampliseq/uppmax.config new file mode 100644 index 0000000..36a1c3b --- /dev/null +++ b/conf/pipeline/ampliseq/uppmax.config @@ -0,0 +1,15 @@ +// Profile config names for nf-core/configs + +params { + // Specific nf-core/configs params + config_profile_contact = 'Daniel Lundin (daniel.lundin@lnu.se)' + config_profile_description = 'nf-core/ampliseq UPPMAX profile provided by nf-core/configs' +} + +withName: make_SILVA_132_16S_classifier { + clusterOptions = { "-A $params.project -C fat -p node -N 1 ${params.clusterOptions ?: ''}" } +} + +withName: classifier { + clusterOptions = { "-A $params.project -C fat -p node -N 1 ${params.clusterOptions ?: ''}" } +} diff --git a/docs/pipeline/ampliseq/uppmax.md b/docs/pipeline/ampliseq/uppmax.md new file mode 100644 index 0000000..7061bdc --- /dev/null +++ b/docs/pipeline/ampliseq/uppmax.md @@ -0,0 +1,17 @@ +# nf-core/configs: uppmax ampliseq specific configuration + +Extra specific configuration for the ampliseq pipeline. + +## Usage + +To use, run the pipeline with `-profile uppmax`. + +This will download and launch the ampliseq specific [`uppmax.config`](../../../conf/pipeline/ampliseq/uppmax.config) which has been pre-configured with a setup suitable for the UPPMAX cluster. + +Example: `nextflow run nf-core/ampliseq -profile uppmax` + +## ampliseq specific configurations for uppmax + +Specific configurations for UPPMAX has been made for ampliseq. + +* Makes sure that a fat node is allocated for training and applying a Bayesian classifier. diff --git a/pipeline/ampliseq.config b/pipeline/ampliseq.config index 6009d68..7b36722 100644 --- a/pipeline/ampliseq.config +++ b/pipeline/ampliseq.config @@ -10,4 +10,8 @@ profiles { binac { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/binac.config" } -} \ No newline at end of file +} + +profiles { + uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/uppmax.config" } +} From 79ef94de4782f17b154dce6821e50c9e36fb8697 Mon Sep 17 00:00:00 2001 From: Daniel Lundin Date: Tue, 31 Mar 2020 15:33:04 +0200 Subject: [PATCH 46/87] Update pipeline/ampliseq.config Co-Authored-By: Maxime Garcia --- pipeline/ampliseq.config | 1 - 1 file changed, 1 deletion(-) diff --git a/pipeline/ampliseq.config b/pipeline/ampliseq.config index 7b36722..167877c 100644 --- a/pipeline/ampliseq.config +++ b/pipeline/ampliseq.config @@ -12,6 +12,5 @@ profiles { binac { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/binac.config" } } -profiles { uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/uppmax.config" } } From 5a557ed7d5b08091eb9f89c6df4c4439c73dbeaa Mon Sep 17 00:00:00 2001 From: Daniel Lundin Date: Tue, 31 Mar 2020 15:33:12 +0200 Subject: [PATCH 47/87] Update pipeline/ampliseq.config Co-Authored-By: Maxime Garcia --- pipeline/ampliseq.config | 1 - 1 file changed, 1 deletion(-) diff --git a/pipeline/ampliseq.config b/pipeline/ampliseq.config index 167877c..c86e152 100644 --- a/pipeline/ampliseq.config +++ b/pipeline/ampliseq.config @@ -11,6 +11,5 @@ profiles { binac { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/binac.config" } } - uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/uppmax.config" } } From e0f9374272c2e97954b611fa3738de70d3e076ba Mon Sep 17 00:00:00 2001 From: Daniel Lundin Date: Tue, 31 Mar 2020 15:33:19 +0200 Subject: [PATCH 48/87] Update pipeline/ampliseq.config Co-Authored-By: Maxime Garcia --- pipeline/ampliseq.config | 1 - 1 file changed, 1 deletion(-) diff --git a/pipeline/ampliseq.config b/pipeline/ampliseq.config index c86e152..9cb19f7 100644 --- a/pipeline/ampliseq.config +++ b/pipeline/ampliseq.config @@ -10,6 +10,5 @@ profiles { binac { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/binac.config" } -} uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/uppmax.config" } } From e77cd380929e33a430ef62925cdbb81dd4f8ed50 Mon Sep 17 00:00:00 2001 From: Daniel Lundin Date: Tue, 31 Mar 2020 15:42:25 +0200 Subject: [PATCH 49/87] Adding ampliseq/UPPMAX doc to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 95c1d79..49283ed 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,7 @@ Currently documentation is available for the following pipeline within the speci * ampliseq * [BINAC](docs/pipeline/ampliseq/binac.md) + * [UPPMAX](docs/pipeline/ampliseq/uppmax.md) * eager * [SHH](docs/pipeline/eager/shh.md) * rnafusion From 34dc18faf6d876a82372e00af7159c954a1dcf34 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Thu, 9 Apr 2020 11:35:37 +0200 Subject: [PATCH 50/87] Update uppmax.config --- conf/pipeline/sarek/uppmax.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/pipeline/sarek/uppmax.config b/conf/pipeline/sarek/uppmax.config index 3feb2df..df2f3a2 100644 --- a/conf/pipeline/sarek/uppmax.config +++ b/conf/pipeline/sarek/uppmax.config @@ -9,9 +9,13 @@ params { igenomeIgnore = true genomes_base = params.genome == 'GRCh37' ? '/sw/data/uppnex/ToolBox/ReferenceAssemblies/hg38make/bundle/2.8/b37' : '/sw/data/uppnex/ToolBox/hg38bundle' } + +def hostname = "hostname".execute().text.trim() + if (hostname ==~ "r.*") { params.singleCPUmem = 6400.MB } + if (hostname ==~ "i.*") { params.singleCPUmem = 15.GB } From 19316ce8527fb1f1f46c842c65116d933779e0bd Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Thu, 9 Apr 2020 13:40:35 +0200 Subject: [PATCH 51/87] add viralrecon specific config --- pipeline/viralrecon.config | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 pipeline/viralrecon.config diff --git a/pipeline/viralrecon.config b/pipeline/viralrecon.config new file mode 100644 index 0000000..3aa23af --- /dev/null +++ b/pipeline/viralrecon.config @@ -0,0 +1,16 @@ +/* + * ------------------------------------------------- + * nfcore/viralrecon custom profile Nextflow config file + * ------------------------------------------------- + * Defines viral reference genomes for all environments. + */ + +params { + // Genome reference file paths + genomes { + 'NC_045512.2' { + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/reference/NC_045512.2.fasta.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/reference/NC_045512.2.gff.gz" + } + } +} From 290d0a21803dc2baee983aa358e7260b97de642c Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 9 Apr 2020 13:31:20 +0100 Subject: [PATCH 52/87] Update pipeline/viralrecon.config --- pipeline/viralrecon.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/viralrecon.config b/pipeline/viralrecon.config index 3aa23af..afe373c 100644 --- a/pipeline/viralrecon.config +++ b/pipeline/viralrecon.config @@ -9,7 +9,7 @@ params { // Genome reference file paths genomes { 'NC_045512.2' { - fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/reference/NC_045512.2.fasta.gz" + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.fasta.gz" gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/reference/NC_045512.2.gff.gz" } } From 4fd9b73cecef10a2e863d88b5bd753ec5a4fd4a7 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 9 Apr 2020 13:31:39 +0100 Subject: [PATCH 53/87] Update pipeline/viralrecon.config --- pipeline/viralrecon.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/viralrecon.config b/pipeline/viralrecon.config index afe373c..45c8247 100644 --- a/pipeline/viralrecon.config +++ b/pipeline/viralrecon.config @@ -10,7 +10,7 @@ params { genomes { 'NC_045512.2' { fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.fasta.gz" - gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/reference/NC_045512.2.gff.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.gff.gz" } } } From 50a44d037bb86703f405c5903ffa944222ce0052 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Thu, 9 Apr 2020 17:02:36 +0200 Subject: [PATCH 54/87] better docs and organisation of giles --- README.md | 12 ++++++++++-- conf/pipeline/viralrecon/genomes.config | 16 ++++++++++++++++ docs/pipeline/viralrecon/genomes.md | 9 +++++++++ pipeline/viralrecon.config | 11 +---------- 4 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 conf/pipeline/viralrecon/genomes.config create mode 100644 docs/pipeline/viralrecon/genomes.md diff --git a/README.md b/README.md index 49283ed..17ce78c 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ A repository for hosting Nextflow configuration files containing custom paramete * [Documentation](#documentation) * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) * [Adding a new pipeline-specific config](#adding-a-new-pipeline-specific-config) + * [Pipeline-specific institutional documentation](#pipeline-specific-institutional-documentation) * [Pipeline-specific documentation](#pipeline-specific-documentation) * [Enabling pipeline-specific configs within a pipeline](#enabling-pipeline-specific-configs-within-a-pipeline) * [Create the pipeline-specific `nf-core/configs` files](#create-the-pipeline-specific-nf-coreconfigs-files) @@ -159,9 +160,9 @@ Each configuration file will add new params and overwrite the params already exi Note that pipeline-specific configs are not required and should only be added if needed. -### Pipeline-specific documentation +### Pipeline-specific institutional documentation -Currently documentation is available for the following pipeline within the specific profile: +Currently documentation is available for the following pipelines within specific profiles: * ampliseq * [BINAC](docs/pipeline/ampliseq/binac.md) @@ -174,6 +175,13 @@ Currently documentation is available for the following pipeline within the speci * [MUNIN](docs/pipeline/sarek/munin.md) * [UPPMAX](docs/pipeline/sarek/uppmax.md) +### Pipeline-specific documentation + +Currently documentation is available for the following pipeline: + +* viralrecon + * [genomes](docs/pipeline/viralrecon/genomes.md) + ### Enabling pipeline-specific configs within a pipeline :warning: **This has to be done on a fork of the `nf-core/` repository.** diff --git a/conf/pipeline/viralrecon/genomes.config b/conf/pipeline/viralrecon/genomes.config new file mode 100644 index 0000000..45c8247 --- /dev/null +++ b/conf/pipeline/viralrecon/genomes.config @@ -0,0 +1,16 @@ +/* + * ------------------------------------------------- + * nfcore/viralrecon custom profile Nextflow config file + * ------------------------------------------------- + * Defines viral reference genomes for all environments. + */ + +params { + // Genome reference file paths + genomes { + 'NC_045512.2' { + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.fasta.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.gff.gz" + } + } +} diff --git a/docs/pipeline/viralrecon/genomes.md b/docs/pipeline/viralrecon/genomes.md new file mode 100644 index 0000000..9577434 --- /dev/null +++ b/docs/pipeline/viralrecon/genomes.md @@ -0,0 +1,9 @@ +# nf-core/configs: viralrecon specific configuration + +Extra specific configuration for viralrecon pipeline + +## Usage + +Will be used automatically when running the pipeline with the shared configs in the nf-core/configs repository + +This will download and launch the viralrecon specific [`viralrecon.config`](../../../conf/pipeline/viralrecon/genomes.config) which has been pre-configured with custom genomes. diff --git a/pipeline/viralrecon.config b/pipeline/viralrecon.config index 45c8247..25d93c4 100644 --- a/pipeline/viralrecon.config +++ b/pipeline/viralrecon.config @@ -2,15 +2,6 @@ * ------------------------------------------------- * nfcore/viralrecon custom profile Nextflow config file * ------------------------------------------------- - * Defines viral reference genomes for all environments. */ -params { - // Genome reference file paths - genomes { - 'NC_045512.2' { - fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.fasta.gz" - gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.gff.gz" - } - } -} +includeConfig "${params.custom_config_base}/conf/pipeline/viralrecon/genomes.config" } \ No newline at end of file From 5678494e45eb098cdc59a05b83c71d6d9d1d63ab Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Thu, 9 Apr 2020 17:13:19 +0200 Subject: [PATCH 55/87] fix typo --- pipeline/viralrecon.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/viralrecon.config b/pipeline/viralrecon.config index 25d93c4..3a79bef 100644 --- a/pipeline/viralrecon.config +++ b/pipeline/viralrecon.config @@ -4,4 +4,4 @@ * ------------------------------------------------- */ -includeConfig "${params.custom_config_base}/conf/pipeline/viralrecon/genomes.config" } \ No newline at end of file +includeConfig "${params.custom_config_base}/conf/pipeline/viralrecon/genomes.config" \ No newline at end of file From 6834f020cb45fdfb1f4bcf181ced275830b4fbb7 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 9 Apr 2020 17:49:15 +0100 Subject: [PATCH 56/87] Update viralrecon.config --- pipeline/viralrecon.config | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pipeline/viralrecon.config b/pipeline/viralrecon.config index 45c8247..d889460 100644 --- a/pipeline/viralrecon.config +++ b/pipeline/viralrecon.config @@ -9,8 +9,12 @@ params { // Genome reference file paths genomes { 'NC_045512.2' { - fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.fasta.gz" - gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.gff.gz" + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.gff.gz" + } + 'MN908947.3' { + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.fna.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz" } } } From 2300d5a9626a486c41248f751d22e39f1ae7e717 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 9 Apr 2020 18:22:41 +0100 Subject: [PATCH 57/87] Update genomes.config --- conf/pipeline/viralrecon/genomes.config | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/viralrecon/genomes.config b/conf/pipeline/viralrecon/genomes.config index 45c8247..d889460 100644 --- a/conf/pipeline/viralrecon/genomes.config +++ b/conf/pipeline/viralrecon/genomes.config @@ -9,8 +9,12 @@ params { // Genome reference file paths genomes { 'NC_045512.2' { - fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.fasta.gz" - gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.gff.gz" + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.gff.gz" + } + 'MN908947.3' { + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.fna.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz" } } } From e6f4d58fa60aa0ae3256d0ab98d5b862573485f9 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 28 Apr 2020 17:06:04 +0200 Subject: [PATCH 58/87] update rnafusion munin specific config --- conf/pipeline/rnafusion/munin.config | 52 +--------------------------- docs/pipeline/rnafusion/munin.md | 1 - pipeline/rnafusion.config | 13 +++++++ 3 files changed, 14 insertions(+), 52 deletions(-) create mode 100644 pipeline/rnafusion.config diff --git a/conf/pipeline/rnafusion/munin.config b/conf/pipeline/rnafusion/munin.config index 549d957..b18f6ad 100644 --- a/conf/pipeline/rnafusion/munin.config +++ b/conf/pipeline/rnafusion/munin.config @@ -4,57 +4,7 @@ params { max_cpus = 24 max_memory = 256.GB max_time = 72.h - container_version = '1.1.0' - - // Tool versions - arriba_version = '1.1.0' - ericscript_version = '0.5.5' - fusioncatcher_version = '1.20' - fusion_inspector_version = '1.3.1-star2.7.0f' - pizzly_version = '0.37.3' - squid_version = '1.5-star2.7.0f' - star_fusion_version = '1.6.0' // Paths - reference_base = '/data1/references/rnafusion' - containerPath = "file:///data1/containers/rnafusion_containers_v${container_version}" - containerPathCommon = "file:///data1/containers/rnafusion_common" - - // References - fasta = "${params.reference_base}/1.1.0/Homo_sapiens.GRCh38_r97.all.fa" - gtf = "${params.reference_base}/1.1.0/Homo_sapiens.GRCh38_r97.gtf" - transcript = "${params.reference_base}/1.1.0/Homo_sapiens.GRCh38_r97.cdna.all.fa.gz" - databases = "${params.reference_base}/1.1.0/databases" - - star_index = "${params.reference_base}/1.1.0/star_index/star" - arriba_ref = "${params.reference_base}/arriba_ref" - fusioncatcher_ref = "${params.reference_base}/fusioncatcher_ref/human_v98" - star_fusion_ref = "${params.reference_base}/star_fusion/1.6.0/ensembl/ctat_genome_lib_build_dir" - ericscript_ref = "${params.reference_base}/ericscript_ref/ericscript_db_homosapiens_ensembl84" -} - -process { - container = "${params.containerPath}/rnafusion_1.1.0.img" - - withName: "arriba|arriba_visualization" { - container = "${params.containerPath}/rnafusion_arriba_v${params.arriba_version}.img" - } - withName: "star_fusion|download_star_fusion" { - container = "${params.containerPath}/rnafusion_star-fusion_v${params.star_fusion_version}.img" - } - withName:fusioncatcher { - container = "${params.containerPath}/rnafusion_fusioncatcher_v${params.fusioncatcher_version}.img" - } - withName:fusion_inspector { - container = "${params.containerPath}/rnafusion_fusion-inspector_v${params.fusion_inspector_version}.img" - } - withName:ericscript { - container = "${params.containerPathCommon}/rnafusion_ericscript_v${params.ericscript_version}.img" - } - withName:pizzly { - container = "${params.containerPathCommon}/rnafusion_pizzly_v${params.pizzly_version}.img" - } - withName:squid { - container = "${params.containerPath}/rnafusion_squid_v${params.squid_version}.img" - } + genomes_base = '/data1/references/rnafusion/dev/' } diff --git a/docs/pipeline/rnafusion/munin.md b/docs/pipeline/rnafusion/munin.md index daa1dd1..607db9e 100644 --- a/docs/pipeline/rnafusion/munin.md +++ b/docs/pipeline/rnafusion/munin.md @@ -15,5 +15,4 @@ Example: `nextflow run nf-core/rnafusion -profile munin` Specific configurations for `MUNIN` has been made for rnafusion. * `cpus`, `memory` and `time` max requirements. -* Paths to specific containers * Paths to specific references and indexes diff --git a/pipeline/rnafusion.config b/pipeline/rnafusion.config new file mode 100644 index 0000000..2d86d89 --- /dev/null +++ b/pipeline/rnafusion.config @@ -0,0 +1,13 @@ +/* + * ------------------------------------------------- + * nfcore/rnafusion custom profile Nextflow config file + * ------------------------------------------------- + * Config options for custom environments. + * Cluster-specific config options should be saved + * in the conf/pipeline/rnafusion folder and imported + * under a profile name here. + */ + +profiles { + munin { includeConfig "${params.custom_config_base}/conf/pipeline/rnafusion/munin.config" } +} \ No newline at end of file From d6813d9c39b8b5458cca8df032957843f554d643 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Mon, 4 May 2020 17:09:43 +0200 Subject: [PATCH 59/87] Add scratch = true to avoid writing of intermediate file to /tmp --- conf/cfc.config | 1 + conf/cfc_dev.config | 1 + 2 files changed, 2 insertions(+) diff --git a/conf/cfc.config b/conf/cfc.config index 62d3626..ffe0395 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -14,6 +14,7 @@ process { beforeScript = 'module load devel/singularity/3.4.2' executor = 'slurm' queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' : 'compute' } + scratch = 'true' } weblog{ diff --git a/conf/cfc_dev.config b/conf/cfc_dev.config index a7ebf2f..6190314 100644 --- a/conf/cfc_dev.config +++ b/conf/cfc_dev.config @@ -13,6 +13,7 @@ process { beforeScript = 'module load devel/singularity/3.4.2' executor = 'slurm' queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' : 'compute' } + scratch = 'true' } weblog{ From d42d7f41463d26f97308a4c6c396e71bf998dd43 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 4 May 2020 21:02:16 +0200 Subject: [PATCH 60/87] Sets the bwa aln parameter to 0.01 which is more commonly used at SHH --- conf/pipeline/eager/shh.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index a34f0e2..22a60d4 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -17,3 +17,7 @@ process { queue = { task.memory > 756.GB ? 'supercruncher' : 'long' } } } + +params { + bwaalnn = 0.01 +} From 4804052b23f607ebeb2ff160abed6d669baa7420 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 4 May 2020 21:04:17 +0200 Subject: [PATCH 61/87] Update SHH nf-core/eager pipeline docs --- docs/pipeline/eager/shh.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/pipeline/eager/shh.md b/docs/pipeline/eager/shh.md index a0dc092..6112842 100644 --- a/docs/pipeline/eager/shh.md +++ b/docs/pipeline/eager/shh.md @@ -15,3 +15,4 @@ Example: `nextflow run nf-core/eager -profile shh` Specific configurations for shh has been made for eager. * If running with the MALT module turned on, the MALT process by default will be sent to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. +* Modifies the bwa aln `-n` parameter to 0.01, rather than typical default of 0.03, to syncronise the (unpublished) parameter with EAGER1 and typical/expected usage within the department. From 4d7e9ccf2f2548f0538d3693e1be31c2084cd2c1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 6 May 2020 08:35:23 +0200 Subject: [PATCH 62/87] Apply suggestions from review --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 3d796ee..755b1d2 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,6 @@ Currently documentation is available for the following systems: * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) -* [EBC](docs/ebc.md) * [DENBI_QBIC](docs/denbi_qbic.md) * [GENOTOUL](docs/genotoul.md) * [GENOUEST](docs/genouest.md) From cf6e697f0b6e4ba6eb7a5ff0671d2a347649b1f1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 7 May 2020 09:16:08 +0200 Subject: [PATCH 63/87] Further update aln based on modifications of EAGER defaults --- conf/pipeline/eager/shh.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 22a60d4..b07004d 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -20,4 +20,5 @@ process { params { bwaalnn = 0.01 + bwaalnl = 32 } From 0a312cd2c91dfa397bcf23b67ac4867fca88e7dc Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 7 May 2020 09:17:56 +0200 Subject: [PATCH 64/87] Update shh.md --- docs/pipeline/eager/shh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pipeline/eager/shh.md b/docs/pipeline/eager/shh.md index 6112842..41d0893 100644 --- a/docs/pipeline/eager/shh.md +++ b/docs/pipeline/eager/shh.md @@ -15,4 +15,4 @@ Example: `nextflow run nf-core/eager -profile shh` Specific configurations for shh has been made for eager. * If running with the MALT module turned on, the MALT process by default will be sent to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. -* Modifies the bwa aln `-n` parameter to 0.01, rather than typical default of 0.03, to syncronise the (unpublished) parameter with EAGER1 and typical/expected usage within the department. +* Modifies the bwa aln `-n` parameter to 0.01, rather than typical default of 0.04, and `-l` to 32 (rather than 1024) to syncronise the (unpublished) parameter with EAGER1 and typical/expected usage within the department. From 2e3b577fb3f49dc574edcbdd55fd3e95d2127dd3 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 7 May 2020 11:50:11 +0200 Subject: [PATCH 65/87] Typo fixes I'll never forgive our directory for that name... --- docs/shh.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/shh.md b/docs/shh.md index c780f21..6f61426 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -2,7 +2,7 @@ All nf-core pipelines have been successfully configured for use on the Department of Archaeogenetic's SDAG/CDAG clusters at the [Max Planck Institute for the Science of Human History (MPI-SHH)](http://shh.mpg.de). -To use, run the pipeline with `-profile ssh`. You can further with optimise submissions by specifying which cluster you are using with `-profile shh,sdag` or `-profile ssh,cdag`. This will download and launch the [`shh.config`](../conf/shh.config) which has been pre-configured with a setup suitable for the SDAG and CDAG clusters respectively. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. The image will currently be centrally stored here: +To use, run the pipeline with `-profile shh`. You can further with optimise submissions by specifying which cluster you are using with `-profile shh,sdag` or `-profile shh,cdag`. This will download and launch the [`shh.config`](../conf/shh.config) which has been pre-configured with a setup suitable for the SDAG and CDAG clusters respectively. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. The image will currently be centrally stored here: ```bash /projects1/singularity_scratch/cache/ @@ -10,7 +10,7 @@ To use, run the pipeline with `-profile ssh`. You can further with optimise subm however this will likely change to a read-only directory in the future that will be managed by the IT team. -This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`) depending on the time and memory required by each process. `-profile ssh,sdag` additionally allows for submission of jobs to the `supercruncher` queue when a job's requested memory exceeds 756GB. +This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`) depending on the time and memory required by each process. `-profile shh,sdag` additionally allows for submission of jobs to the `supercruncher` queue when a job's requested memory exceeds 756GB. >NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact the IT team. >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. From 4814bac1c342e9715539a28a0967b33d3df5e86d Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 12 May 2020 12:42:42 +0200 Subject: [PATCH 66/87] Fix sub-profiles --- conf/shh.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/shh.config b/conf/shh.config index a6106be..852f1d3 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -33,11 +33,15 @@ params { profiles { cdag { + params { config_profile_description = 'MPI-SHH CDAG profile, provided by nf-core/configs.' + } } sdag { + params { config_profile_description = 'MPI-SHH SDAG profile, provided by nf-core/configs.' max_memory = 2.TB max_cpus = 128 + } } } From 8225e07819fbfc76aa1a4048e12d740c52181aff Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 12 May 2020 12:47:25 +0200 Subject: [PATCH 67/87] Condensed base params and added names for debugging --- conf/shh.config | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index 852f1d3..f10f1f8 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -1,8 +1,13 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'Generic MPI-SHH cluster(s) profile provided by nf-core/configs.' + config_profile_description = 'JFY Generic MPI-SHH cluster(s) profile provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' config_profile_url = 'https://shh.mpg.de' + max_memory = 256.GB + max_cpus = 32 + max_time = 720.h + //Illumina iGenomes reference file path + igenomes_base = "/projects1/public_data/igenomes/" } cleanup = true @@ -23,23 +28,15 @@ executor { queueSize = 16 } -params { - max_memory = 256.GB - max_cpus = 32 - max_time = 720.h - //Illumina iGenomes reference file path - igenomes_base = "/projects1/public_data/igenomes/" -} - profiles { cdag { params { - config_profile_description = 'MPI-SHH CDAG profile, provided by nf-core/configs.' + config_profile_description = 'JFY MPI-SHH CDAG profile, provided by nf-core/configs.' } } sdag { params { - config_profile_description = 'MPI-SHH SDAG profile, provided by nf-core/configs.' + config_profile_description = 'JFY MPI-SHH SDAG profile, provided by nf-core/configs.' max_memory = 2.TB max_cpus = 128 } From ff915ee5045345075b854712694408a7e555c0ce Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 12 May 2020 12:47:59 +0200 Subject: [PATCH 68/87] Update shh.config --- conf/pipeline/eager/shh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index b07004d..1ed9112 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -3,7 +3,7 @@ params { // Specific nf-core/configs params config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_description = 'nf-core/eager SHH profile provided by nf-core/configs' + config_profile_description = 'JFY nf-core/eager SHH profile provided by nf-core/configs' igenomes_base = "/projects1/public_data/igenomes/" } From e31e781f2938fc95adc91b4c22f79eacbc4e56e8 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 May 2020 11:07:06 +0200 Subject: [PATCH 69/87] Remove debugging --- conf/shh.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index f10f1f8..8defc1f 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -1,6 +1,6 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'JFY Generic MPI-SHH cluster(s) profile provided by nf-core/configs.' + config_profile_description = 'Generic MPI-SHH cluster(s) profile provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' config_profile_url = 'https://shh.mpg.de' max_memory = 256.GB @@ -31,12 +31,12 @@ executor { profiles { cdag { params { - config_profile_description = 'JFY MPI-SHH CDAG profile, provided by nf-core/configs.' + config_profile_description = 'CDAG MPI-SHH profile, provided by nf-core/configs.' } } sdag { params { - config_profile_description = 'JFY MPI-SHH SDAG profile, provided by nf-core/configs.' + config_profile_description = 'SDAG MPI-SHH profile, provided by nf-core/configs.' max_memory = 2.TB max_cpus = 128 } From d070ed70d18bc3766fb71d053d898e022aae04cb Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 May 2020 11:11:42 +0200 Subject: [PATCH 70/87] Update BWA settings based on feedback --- conf/pipeline/eager/shh.config | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 1ed9112..94a0403 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -3,8 +3,12 @@ params { // Specific nf-core/configs params config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_description = 'JFY nf-core/eager SHH profile provided by nf-core/configs' + config_profile_description = 'nf-core/eager SHH profile provided by nf-core/configs' igenomes_base = "/projects1/public_data/igenomes/" + + // default BWA + bwaalnn = 0.04 + bwaalnl = 32 } // Specific nf-core/eager process configuration @@ -18,7 +22,26 @@ process { } } -params { - bwaalnn = 0.01 - bwaalnl = 32 +profiles { + pathogen_loose { + params { + config_profile_description = 'Pathogen (loose) MPI-SHH profile, provided by nf-core/configs.' + bwaalnn = 0.01 + bwaalnl = 16 + } + } + pathogen_strict { + params { + config_profile_description = 'Pathogen (strict) MPI-SHH SDAG profile, provided by nf-core/configs.' + bwaalnn = 0.1 + bwaalnl = 32 + } + } + human { + params { + config_profile_description = 'Human MPI-SHH SDAG profile, provided by nf-core/configs.' + bwaalnn = 0.01 + bwaalnl = 16500 + } + } } From 0943ba088a2fdd23aabccb73ba96b41001f7d2de Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 May 2020 11:14:42 +0200 Subject: [PATCH 71/87] Mention SHH EAGER group-specific profiles --- docs/pipeline/eager/shh.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pipeline/eager/shh.md b/docs/pipeline/eager/shh.md index 41d0893..d5fbcb3 100644 --- a/docs/pipeline/eager/shh.md +++ b/docs/pipeline/eager/shh.md @@ -14,5 +14,5 @@ Example: `nextflow run nf-core/eager -profile shh` Specific configurations for shh has been made for eager. -* If running with the MALT module turned on, the MALT process by default will be sent to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. -* Modifies the bwa aln `-n` parameter to 0.01, rather than typical default of 0.04, and `-l` to 32 (rather than 1024) to syncronise the (unpublished) parameter with EAGER1 and typical/expected usage within the department. +* If running with the MALT module turned on, the MALT process by default will be sent to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. Note, this will only work on SDAG. +* Provides additional group specific profiles, which adapt the `bwa aln` mapping parameters to each context: `pathogens_loose` (`-l 0.01 -n 16`), `pathogens_strict` (`-l 32, -n 0.1`) and `human` (`-l 16500, -n 0.01`). From 1354ae40635dbf130f92df22d93a01e70b07c745 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:19:15 +0200 Subject: [PATCH 72/87] add bi config --- .github/workflows/main.yml | 2 +- conf/bi.config | 11 +++++++++++ nfcore_custom.config | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 conf/bi.config diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bdc485b..b927957 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/conf/bi.config b/conf/bi.config new file mode 100644 index 0000000..874d9f8 --- /dev/null +++ b/conf/bi.config @@ -0,0 +1,11 @@ +params.globalConfig = determine_global_config() + +includeConfig params.globalConfig + +def determine_global_config() { + if( System.getenv('NXF_GLOBAL_CONFIG') == null) + { + throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") + } + return System.getenv('NXF_GLOBAL_CONFIG') +} diff --git a/nfcore_custom.config b/nfcore_custom.config index ea2238a..3f89d83 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -11,6 +11,7 @@ //Please use a new line per include Config section to allow easier linting/parsing. Thank you. profiles { awsbatch { includeConfig "${params.custom_config_base}/conf/awsbatch.config" } + bi { includeConfig "${params.custom_config_base}/conf/bi.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } cbe { includeConfig "${params.custom_config_base}/conf/cbe.config" } From c31b03108c4308c70a6f4ac68840ad0cbe37e308 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:22:56 +0200 Subject: [PATCH 73/87] retry without Exception --- conf/bi.config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/bi.config b/conf/bi.config index 874d9f8..091092c 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -3,9 +3,9 @@ params.globalConfig = determine_global_config() includeConfig params.globalConfig def determine_global_config() { - if( System.getenv('NXF_GLOBAL_CONFIG') == null) - { - throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") - } + // if( System.getenv('NXF_GLOBAL_CONFIG') == null) + // { + // throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") + // } return System.getenv('NXF_GLOBAL_CONFIG') } From 2685275952076be368b7103b0f49bf11f2ed1aac Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:24:47 +0200 Subject: [PATCH 74/87] switch include off --- conf/bi.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/bi.config b/conf/bi.config index 091092c..6fa20d6 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -1,6 +1,6 @@ params.globalConfig = determine_global_config() -includeConfig params.globalConfig +// includeConfig params.globalConfig def determine_global_config() { // if( System.getenv('NXF_GLOBAL_CONFIG') == null) From 66f191a1cfd87ac5b5b728a0ebc73b84b9d469c9 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:30:00 +0200 Subject: [PATCH 75/87] declate NXF_GLOBAL_CONFIG in ci --- .github/workflows/main.yml | 2 ++ conf/bi.config | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b927957..8226dac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,9 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ + touch /usr/local/bin/test.config - name: Check ${{ matrix.profile }} profile env: SCRATCH: '~' + NXF_GLOBAL_CONFIG: /usr/local/bin/test.config run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} diff --git a/conf/bi.config b/conf/bi.config index 6fa20d6..874d9f8 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -1,11 +1,11 @@ params.globalConfig = determine_global_config() -// includeConfig params.globalConfig +includeConfig params.globalConfig def determine_global_config() { - // if( System.getenv('NXF_GLOBAL_CONFIG') == null) - // { - // throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") - // } + if( System.getenv('NXF_GLOBAL_CONFIG') == null) + { + throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") + } return System.getenv('NXF_GLOBAL_CONFIG') } From ab3070582e94ded2270dcacd69b1348aceec488f Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:32:19 +0200 Subject: [PATCH 76/87] Fix paths to dummy.config --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8226dac..8c6d23b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,9 +23,9 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - touch /usr/local/bin/test.config + touch ~/dummy.config - name: Check ${{ matrix.profile }} profile env: SCRATCH: '~' - NXF_GLOBAL_CONFIG: /usr/local/bin/test.config + NXF_GLOBAL_CONFIG: ~/dummy.config run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From ff5fdd097bf418b1cb78a7a171c1fb9a4744ee59 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:35:55 +0200 Subject: [PATCH 77/87] debug --- .github/workflows/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8c6d23b..d4e0232 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,9 +23,13 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ + echo ~ touch ~/dummy.config - name: Check ${{ matrix.profile }} profile env: SCRATCH: '~' NXF_GLOBAL_CONFIG: ~/dummy.config - run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} + run: | + echo $SCRATCH + echo $NXF_GLOBAL_CONFIG + nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From ac47d020cb8f09dac8ceae1ca3e2f86c3ed591f7 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:38:11 +0200 Subject: [PATCH 78/87] debug: continue --- .github/workflows/main.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d4e0232..8c50e7a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,12 +23,11 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - echo ~ - touch ~/dummy.config + touch /home/runner/dummy.config - name: Check ${{ matrix.profile }} profile env: SCRATCH: '~' - NXF_GLOBAL_CONFIG: ~/dummy.config + NXF_GLOBAL_CONFIG: /home/runner/dummy.config run: | echo $SCRATCH echo $NXF_GLOBAL_CONFIG From 1a9db371c380e4c1a044a3586bb33ac492f714bd Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:43:32 +0200 Subject: [PATCH 79/87] remove echos --- .github/workflows/main.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8c50e7a..c906cf9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,4 @@ jobs: env: SCRATCH: '~' NXF_GLOBAL_CONFIG: /home/runner/dummy.config - run: | - echo $SCRATCH - echo $NXF_GLOBAL_CONFIG - nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 3ad0003979a4d7c598be03980d1640c9327a8478 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:46:08 +0200 Subject: [PATCH 80/87] test without dummy config --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c906cf9..0742e5c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,9 +23,8 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - touch /home/runner/dummy.config - name: Check ${{ matrix.profile }} profile env: SCRATCH: '~' - NXF_GLOBAL_CONFIG: /home/runner/dummy.config + NXF_GLOBAL_CONFIG: ${GITHUB_WORKSPACE}/config/awsbatch.config run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From f5e2a9f9f3443ae4be1885cd816ae0bc62cc27fc Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:55:37 +0200 Subject: [PATCH 81/87] try without dummy.config: use relative path --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0742e5c..0609a85 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,5 +26,5 @@ jobs: - name: Check ${{ matrix.profile }} profile env: SCRATCH: '~' - NXF_GLOBAL_CONFIG: ${GITHUB_WORKSPACE}/config/awsbatch.config + NXF_GLOBAL_CONFIG: awsbatch.config run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 18d8edb0469c81838ddd4454b2d410b3807fe1cc Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 17:41:19 +0200 Subject: [PATCH 82/87] add description and profile --- conf/bi.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/bi.config b/conf/bi.config index 874d9f8..0ad15d4 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -1,3 +1,8 @@ +params{ + config_profile_description = 'Boehringer Ingelheim internal profile provided by nf-core/configs.' + config_profile_contact = 'Alexander Peltzer (@apeltzer)' + config_profile_url = 'https://www.boehringer-ingelheim.com/' +} params.globalConfig = determine_global_config() includeConfig params.globalConfig From 3c695ba71c87955f9303151b3b77cab64325fe40 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 18:00:47 +0200 Subject: [PATCH 83/87] test error display --- conf/bi.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/bi.config b/conf/bi.config index 0ad15d4..b96c8e9 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -10,6 +10,7 @@ includeConfig params.globalConfig def determine_global_config() { if( System.getenv('NXF_GLOBAL_CONFIG') == null) { + System.err.println("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") } return System.getenv('NXF_GLOBAL_CONFIG') From f139750bf442aefcefe6d9588206037c92757ba5 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 19:14:43 +0200 Subject: [PATCH 84/87] print the error message before throwing Exception --- conf/bi.config | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conf/bi.config b/conf/bi.config index b96c8e9..f9dfb27 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -10,8 +10,9 @@ includeConfig params.globalConfig def determine_global_config() { if( System.getenv('NXF_GLOBAL_CONFIG') == null) { - System.err.println("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") - throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") + def errorMessage = "ERROR: Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file." + System.err.println(errorMessage) + throw new Exception(errorMessage) } return System.getenv('NXF_GLOBAL_CONFIG') } From 0efa9f248ee616bf36347cb37edcb817bbd57f4c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 28 May 2020 10:12:50 +0200 Subject: [PATCH 85/87] Add BI to main readme.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 17ce78c..e5b199c 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ Currently documentation is available for the following systems: * [AWSBATCH](docs/awsbatch.md) * [BIGPURPLE](docs/bigpurple.md) +* [BI](docs/bi.md) * [BINAC](docs/binac.md) * [CBE](docs/cbe.md) * [CCGA_DX](docs/ccga_dx.md) From f4df5b5a75d4f8541fe04e19aff0e12253bde6a9 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 28 May 2020 10:16:22 +0200 Subject: [PATCH 86/87] Add BI Docs --- docs/bi.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/bi.md diff --git a/docs/bi.md b/docs/bi.md new file mode 100644 index 0000000..b256805 --- /dev/null +++ b/docs/bi.md @@ -0,0 +1,9 @@ +# nf-core/configs: BI Configuration + +All nf-core pipelines have been successfully configured for use at Boehringer Ingelheim. + +To use, run the pipeline with `-profile bi`. This will download and launch the [`bi.config`](../conf/bi.config) which has been pre-configured with a setup suitable for the BI systems. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to follow the internal documentation to run Nextflow on our systems. Similar to that, you need to set an environment variable `NXF_GLOBAL_CONFIG` to the path of the internal global config which is not publicly available here. + +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From dd56355c7ccb2732e9afaaafd07a818f8f1143f3 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 28 May 2020 10:18:16 +0200 Subject: [PATCH 87/87] Update bi.md No trailing spaces it said. --- docs/bi.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/bi.md b/docs/bi.md index b256805..450d0df 100644 --- a/docs/bi.md +++ b/docs/bi.md @@ -2,8 +2,8 @@ All nf-core pipelines have been successfully configured for use at Boehringer Ingelheim. -To use, run the pipeline with `-profile bi`. This will download and launch the [`bi.config`](../conf/bi.config) which has been pre-configured with a setup suitable for the BI systems. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile bi`. This will download and launch the [`bi.config`](../conf/bi.config) which has been pre-configured with a setup suitable for the BI systems. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. -Before running the pipeline you will need to follow the internal documentation to run Nextflow on our systems. Similar to that, you need to set an environment variable `NXF_GLOBAL_CONFIG` to the path of the internal global config which is not publicly available here. +Before running the pipeline you will need to follow the internal documentation to run Nextflow on our systems. Similar to that, you need to set an environment variable `NXF_GLOBAL_CONFIG` to the path of the internal global config which is not publicly available here. >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT.