From 6c5a5c8bcb5b548c00cb25a5ee9016853a809603 Mon Sep 17 00:00:00 2001 From: Marc Hoeppner Date: Mon, 24 Feb 2020 12:01:43 +0100 Subject: [PATCH 01/22] Removing old CCGA profile from docs and linting Removing CCGA profile from master config Removing CCGA profile from master config --- .github/workflows/main.yml | 2 +- README.md | 1 - conf/ccga.config | 41 -------------------------------------- docs/ccga.md | 18 ----------------- nfcore_custom.config | 1 - 5 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 conf/ccga.config delete mode 100644 docs/ccga.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5fb2601..6f96a98 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/README.md b/README.md index 3011a00..7634409 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,6 @@ Currently documentation is available for the following systems: * [BIGPURPLE](docs/bigpurple.md) * [BINAC](docs/binac.md) * [CBE](docs/cbe.md) -* [CCGA](docs/ccga.md) * [CCGA_DX](docs/ccga_dx.md) * [CCGA_MED](docs/ccga_med.md) * [CFC](docs/binac.md) diff --git a/conf/ccga.config b/conf/ccga.config deleted file mode 100644 index 6163626..0000000 --- a/conf/ccga.config +++ /dev/null @@ -1,41 +0,0 @@ -//Profile config names for nf-core/configs -params { - config_profile_description = 'CCGA cluster profile provided by nf-core/configs.' - config_profile_contact = 'Marc Hoeppner (@marchoeppner)' - config_profile_url = 'https://www.ccga.uni-kiel.de/' -} - -/* - * ------------------------------------------------- - * Nextflow config file for CCGA cluster in Kiel - * ------------------------------------------------- - */ - -singularity { - enabled = true - runOptions = "-B /ifs -B /scratch -B /work_beegfs" - cacheDir = "/ifs/data/nfs_share/ikmb_repository/singularity_cache/" -} - -executor { - queueSize=100 -} - -process { - - // Global process config - executor = 'slurm' - queue = 'ikmb_a' - - clusterOptions = { "--qos=ikmb_a" } - -} - -params { - // illumina iGenomes reference file paths on RZCluster - igenomes_base = '/ifs/data/nfs_share/ikmb_repository/references/iGenomes/references/' - saveReference = true - max_memory = 128.GB - max_cpus = 16 - max_time = 120.h -} diff --git a/docs/ccga.md b/docs/ccga.md deleted file mode 100644 index 798df29..0000000 --- a/docs/ccga.md +++ /dev/null @@ -1,18 +0,0 @@ -# nf-core/configs: CCGA Configuration - -Deployment and testing of nf-core pipelines at the CCGA cluster is on-going. - -To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. - -Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on the cluster. You can do this by issuing the commands below: - -```bash -## Load Nextflow and Singularity environment modules -module purge -module load IKMB -module load Java/1.8.0 -module load Nextflow -module load singularity3.1.0 -``` - ->NB: Access to the CCGA cluster is restricted to IKMB/CCGA employes. Please talk to Marc Hoeppner to get access (@marchoeppner). diff --git a/nfcore_custom.config b/nfcore_custom.config index 8b4a3a9..0fcbe66 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -17,7 +17,6 @@ profiles { bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } cbe { includeConfig "${params.custom_config_base}/conf/cbe.config" } - ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } ccga_med { includeConfig "${params.custom_config_base}/conf/ccga_med.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } From e2e4dd05d318c6a537f5dd17f6a1ec82c5a32167 Mon Sep 17 00:00:00 2001 From: phue Date: Thu, 5 Mar 2020 16:34:20 +0100 Subject: [PATCH 02/22] cbe: move container cache to different filesystem this is a workaround for squashfs errors that occur when containers are run from beeGFS --- conf/cbe.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cbe.config b/conf/cbe.config index ea24e5b..5888202 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -14,7 +14,7 @@ process { singularity { enabled = true - cacheDir = '/scratch-cbe/shared/containers' + cacheDir = '/resources/containers' } params { From 8a9d3ad8a0ca66ef35276adacc1facac4d1b0129 Mon Sep 17 00:00:00 2001 From: Evan Floden Date: Fri, 6 Mar 2020 05:47:29 +0000 Subject: [PATCH 03/22] First commit of Google Cloud config --- .github/workflows/main.yml | 2 +- README.md | 1 + conf/google.config | 21 +++++++++++++++++++++ docs/google.md | 30 ++++++++++++++++++++++++++++++ docs/images/google-cloud-logo.png | Bin 0 -> 13226 bytes nfcore_custom.config | 1 + 6 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 conf/google.config create mode 100644 docs/google.md create mode 100644 docs/images/google-cloud-logo.png diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5fb2601..d68f44f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'google', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/README.md b/README.md index 0e58b09..b49edb7 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,7 @@ Currently documentation is available for the following systems: * [GENOTOUL](docs/genotoul.md) * [GENOUEST](docs/genouest.md) * [GIS](docs/gis.md) +* [GOOGLE](docs/google.md) * [HEBBE](docs/hebbe.md) * [KRAKEN](docs/kraken.md) * [MUNIN](docs/munin.md) diff --git a/conf/google.config b/conf/google.config new file mode 100644 index 0000000..4333d86 --- /dev/null +++ b/conf/google.config @@ -0,0 +1,21 @@ +// Nextflow config file for running on Google Cloud Life Sciences +params { + config_profile_description = 'Google Cloud Life Sciences Profile' + config_profile_contact = 'Evan Floden, Seqera Labs (@evanfloden)' + config_profile_url = 'https://cloud.google.com/life-sciences' + + google_zone = 'europe-west2-c' + google_bucket = false + google_debug = false + google_preemptible = true +} + +process.executor = 'google-lifesciences' +google.zone = params.google_zone +google.lifeSciences.debug = params.google_debug +workDir = params.google_bucket +google.lifeSciences.preemptible = params.google_preemptible +if (google.lifeSciences.preemptible) { + process.errorStrategy = { task.exitStatus==14 ? 'retry' : 'terminate' } + process.maxRetries = 5 +} \ No newline at end of file diff --git a/docs/google.md b/docs/google.md new file mode 100644 index 0000000..d2b0f51 --- /dev/null +++ b/docs/google.md @@ -0,0 +1,30 @@ +# nf-core/configs: Google Cloud Life Sciences Configuration +![Google Cloud](images/google-cloud-logo.png "https://cloud.google.com/life-sciences/docs/tutorials/nextflow") + +To be used with the `google` profile by specifying the `-profile google` when running nf-core pipelines. + +## Quick Start +``` +GOOGLE_APPLICATION_CREDENTIALS=.json +NXF_MODE=google +nextflow run nf-core/rnaseq -profile google,test --google_bucket +``` + +## Required Parameters +#### `--google-bucket` +The Google Cloud Storage bucket location to be used as a Nextflow work directory. Can also be specified with (`-w gs://your_bucket/work`). + +## Optional Parameters +#### `--google-zone` +The Google zone where the computation is executed in Compute Engine VMs. Multiple zones can be provided separating them by a comma. Default (`europe-west2-c`). + +#### `--google-preemptible` +Enables the usage of preemptible virtual machines with a retry error statergy for up to 5 retries. Default (`true`) + +#### `--google-debug` +Copies the /google debug directory from the VM to the task bucket directory. Useful for debugging. Default (`false`). + +## Google Cloud Life Sciences Setup +Please refer to the [Google Cloud](https://cloud.google.com/life-sciences/docs/tutorials/nextflow) and [Nextflow](https://www.nextflow.io/docs/latest/google.html#cloud-life-sciences) documentation which describe how to setup the Google Cloud environment. + + diff --git a/docs/images/google-cloud-logo.png b/docs/images/google-cloud-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..07c9a325b254d93f3fe4e94abb5cbdd9162afd92 GIT binary patch literal 13226 zcmZ{LWmFtZ(C#em?hxEvg9ZyO!QI^@xVuYm4=%yoC1}tM9^5SicY=SL_ulWE`{$lL zXJ=-&d#0zlx}K`1s-u-ZNuweWAprn@Dk~$Q3IL!R$oVG(ILP;Q0N3;+U|0D$lOwNph1@&}x$ytD-H z_U}{B^DPAcIQwNK#MHf3P7S>M)HLqi@}81wmEYs5zsGzhZJ`2#uReZsc4)m;peQ4k zt$ExYPjv_$LiN<2e$S$qjdNeQ`-MDb_x&r*y}Q1su{kCNJ&Qw%=hS(EX4w048bw2` zjeD-YS?}w3@AEv*U-DefI(}maP%I%g?61#Amtzt^570xQ!a1`CiIJOg3y0+a>Oc<2 z9_~_#-Ct>IihAxhwVjBd@sA8alYuZqY+8jA=z_`Fq+2V}u3l9q{>Z4;p)Ew9Fl+pts3YF0?dq9pjU|kyJuiE?tw!fxd z<4?a^T7^t@lL6`*>IRdeChZ`AD4!w>skt1#KanwC-N zgjUBRs;j6=bl^qM>!d4l?3s78R^UKbnn5-Ag?%v6-c>g=74>ea~W)aCl zvE(Fkl6wu&QEbbBW06eYivJ^!PArL>q=&HG5zO5lR*3_^8Q@}B=Kg%L`M(YByu~-U zZK~geq=WbhkjEt+DyO0Zzpb^N%I0LKff-Ey6Kf|Fif4z@SE2VAnM^CQk52M_zkG4d zh}`LTqqt3nkKGBg0t3!3fV{kJF#9offfA!yC*J!|LzcW8P=Fq7h_i7%x~>AE*CT+ z3Ek#|&-TdVq#KNx&uaW@jHrdxVqUNP$1b>;MicC0=A*k8Is zuuCNptn(2W0+Hb4d$~%@2PELJ9;dJq-q}-3L;`omG2(wa?-ABMT-Ek4U(<(l2K-U{ zaJGq08~)Q$ehhrXYXHTYcK?*SWitk!FN|(UqkW~<%JwatD3}-yFV)W0 zJix0>K5r|h)mH-pq*=FDm~bbjsUp)z1Mca~7vWr%c%P~=jh<^eNE9=dkoVm1NQhMa z(yKm*h8#|WkxF#Sb8=vMU`>!|Z=Z`>Scqiqa3pR{3sI<|Rme%vjfJMwxeVa*?>~^*?%s)$tMC+Tub7o&?pAQI`76on5VATs@JpXOu+y6QW6*1h!s;8-GP}Zbdqi z#R4!1;wgl{K=9085h(SL^?e=mVI*Mcm!^|pcD%;{whw{bST4jA&?Q~rTEFamuH^QP0~xKlIfahLhN2e1ly28=O- zx&jG#YriSfh>%HWVX?no+4NsH_geE*s2cOT&(3j^cHf`P@nJnj!=|!|c!H6+)Y)DcNtGL|zy{vWL0wT!e06CSk7=3vtKB-I7{^eJ^-b?Y)}=_ny?VeXr-; zYN*DbAXs9$ge(Q{Jv8?KIj2+h^-a(zA*q2oM}j7v9 zmreoz$Cy5!()F}J-_tKO=;|686=ls`B3P6+MorSUXwXe4r--|Xc+RBld!c0UwpV>L zWX#(qCo?&zD|U!}Pv2gKb`9hHvxJU*98M+F5XqfCzMBdL+jIJeRl0`iaoX- zHjSh~0zaO<)gr$UI~_tiy`tX%vXOU{L)lhhY5pd;$$>F1($)DMzT>SxE zYfm3m?oIJ~$*PvYRB#kDNzWmY4o&Ju+X9kyc zc7PvV_vbZ(2?T8~Q83AHrG2EqZMr4Pc{bz2=9!&MyAsHN_H$N@`l1vSF;2 zcKmom)+R=%4BdD*#C5wdSKP@0+xH7oyAG*07+v()R-#2HJ-}zZvX$h*cX771V&mwV zO4AQm=UQ)fUI|iUG|P~(P%b4nk1BGJ~LbMY8@8z#uHh5qP%n}&fcGDEwpdA|VP z*Vi#*k5EQ9J3Fkv#$a2rOTYHJLObuU*)7;D7?btROIcIXGWp+NzRfMC;QHXMEOWPU z?oJg3onMViOr+4RCid?As!xl+HdARzSYS|Cwk)JhnFaw#2&|Y{CgXfos6P9Fp{ba9 z*fgJWUoZFp7BsdNRvT^0cgvd4MtApl69A)P>pNP3UZ(Ct1m;C@DIY^q91S?R^vnr} zN3dHEGcOI0)HWPxGAPh#!TG&$UJ2&YGd%G8pqzr;zAQ7U(aapqxy4ox-;4)e2M7op z$JE)XMLNN*bY2jcVafX87@GP=WsLZ?RJzxPmEw!KC%=#g|C!9@$bF#cT0b)cYU4P* zScQT%{aj@sKu6+YsNm{Dp! zG4&l-;jwAsLLoR#5#6C{_bHig5CPwpSrz@kKRnIC!VdgT-@I&Awq$Z|p`lp+^aYU8 z^qq%gXwwp}*E6PaQU|8zjrD~@d(Pu{4tM=Q8J~Ijir~t`prAu)-5t=)~q`_xS(DURutWlD%7@7T3UkRpLFUxEoVyJJR`$C(ZCC*(lCPO9EHq2=A@)w zPLYEe^@)T^Bm#$hS}R4OaX&59W_@ozL{ck`te{%SQ21cHX?~2&ZtrTSOgD~L|Fb+4 zhL4C4{`}o@MO>-y&X#!YuIv?w?^-Fy>%V-vl3 z9JGlMsHdu05k3NqZPr?)xg@AWx-tN>7B zuaaOGw9e(j_TBJM(n8oQ4-f%VqZW@rka7Wg(7P5aNFOagB;+qkMGQU~vEP^!sH=SebKZ2=Q0B7I~_`6A_H%o zF%icm)bMos@lR|ta%0+{EHthDC-UzNJh6J4X4`~Nu;`r5r1Z5h(S*kO3e>EvT|ycq zF;yY`JE$;^@O>&wx8Z{opk|fu+O;gM1acd1ge|U}bXsHTx9sB^FmB`7nKAp4 zlx+N=YCO6j;?KaF3IE_xS4*5YeDO)mel=pPqyz3_Xk+e7Ddc{wI^C{qc*tbEs#8qp z`L<}#yi$zQ-oFc?!g@FI#URN{^x5QkGZsz=aqDVGKYAOB2FGS-)5 zjI(0!kdMJ$rNdcJek|8KZz-B2BZrW^$SeTM87>j567R6ec5MepZAO20oIU-O;nrS) z;G^|6?`$xhvu**M^}dFFV*3gP2lQ@UaYEUCPd zUOKYAf{Ul5=5BA@hIPhULh-OV3#ZX*yZn7;H|mJ}Oh9(FD0XD9iar@;KK~Wcr&Bb} z&wL~{Xc;4msdt@ezS!}m)8dg6*v-qevhV=q3Mqb}bh>UW=u#(FPw2U*kw@QJ-Hz#+ zA#RvEX8107ROGVNA<{iH74o!c8q5K$1QB5;>3ePVhe*BI0VcwF;-wRt>Wh(%$&j!R zj?>y_S;z%|vX*M?)5EAKkP90qBlF2SPf!#mhAJbldfA_z=|(uH`c7Txbn;mm;f6j) z9eRaWvGVWqA22WX=8uHq_oQ>??(Z{}TUXxYY>M(mtnU-j4g7@(i$!6V`eJ_YPhi!x z%qbJdNRRc&tzUzmVQ>#=OG)qIstxKo*3VEZC1`GX^v|B99t#jC+dLdEr>oKl#bib-B-$KM^lOt_*z zPhe7g{uFXu!;9H_Fgnr8t!?<|w>I>LkuSer30lSm`ZGxxR&P#@7JFf1_g>RNB0rZU z@lt~DsRAB4bo;=T<-Jxko}wpi-c@GrQsS+s;xc!!^#OAt`A$wYI?VGQf91R-2fkMbC=uwNxAeTEIfN^)I9eg( zLV}kY{o(NOpv%OfjP9iNxq>3_Pp=uaReuHeS)DlP!Rb&Q9BT=tCm~kqOBl>DGH1_f ztAiiTG8$?~rsB!x6otmnQmur>9nElYaTzAS9GglUq>rJHf_htYV7cxNnK4%XL4M6+ z9FPU8i4YYZ--a=KsZFK6#wlJuDbAT<~aD8&5$ddY+CDKS^7>BY-&2vO+2 zVP4VzrZj3tY~g+atH&Q4!WXN&iz)ww544>kXoQl0;c!6)>|fushAeWQls;wl{PjH> zIh%9ObLaL*u9=?CLXRJU9!f?tYy)0yM z?!oEQINgEj&mY;Frs_)?gl|l#B6q(0%p1DsNFQ>rvl-t4;D$ZZ)JZ+qLkFP)II9ME zpMe7qpXRrkPze}UkhKq`MCLjC;TFNl7(9kL#!W_b+=fv%>ei>OIkV6{{rF0q-t7`G zy9Fayi7A-_kFJ_q#s`r0Z3Y1z=5-4Mz>I8!RNt>_@F|E2ONe$2FV$0$Mi-JxM%j48 zNcz|Fo4|zrAO^Zowk*0PxX|K$ixKVT2NQOD^8Asvc!d=rdT#xNgQEVsV6B2ILmOy1 z2KSM-Vkf!mz981vdn~Zq@VugG!Lgfo%bGU~hU-~UQc~LeuIu<8sbJ}yFg%hIeK>WA z(|`5!q6gB?%E@?x3(kvk|4I=wEchG9wJH~b?e6b0(l(6?X9;DI`%8|;mqh5u&f++) zykmRR(MszIC`UV)mt99+@B(je#@5N0W9_x;_(TIO9L750ppyxO{yWf##?>hi&4l`4%xQZ)(u7r`&fR zN6BIa@+*HLihlL>789GlJ`qT>gmQC3aliX++~-+_mIV!vUgHNE{khNm*!S`DcEdFn z)(VPG@pqB{gYzeLWf!MvrcV^T=kHX9_yv5?F{6D^mR?kg&p{+&yn}99+w5@7P^p^} z2B!23brVF6oP*xLeN*q=H)>H>S(QKTTSNa%hIozhpgP4EuSe8EP&Hs>a z3q2Ds?RGird#zp2#t-hJ64ApDepjF&kYJ&6q-VI>yljsYY|4;K^Y3jj1M4y2U%pqF zF$%+p1-%tj{RVO2f$v#j0x2ac`#bv!==KHHzfpJH6iu1TltOjx1P>hjQt1(iQ^btX z|5{G-!CdrB55yeDC@r?QV&9rN$(TaBFL2~QY?+7q)#zD_@>CN}4iD=vI5P~bXabcG zagHUuZ9q+}EJ7su)_P^=(WxedcoOy+H7ZZOXS}ExJxh9Uc3J1mz(cJJHEsfQ!ddh) zN_o<0iUWnHc@r+taPv=8ZR2P5^AFOQ-Vu|#sOCxd)Gjo2m6dW1%6p~8hZaEsY`pg| z0lu#T`!2VzZf430Wx9Je5}d{lFRwM?mH2-)-rv-*1gLkJP5eUSzze^g z@iL#W9G#dDOhok2g8K+TqYCX)bKf4kt~pne+PS7wjZ0hv=_96uol5F1QuisoXn=pN|cJ1qW+Y)2g+XwG}TzvI)C2z=f81vbJImI>v~wZtpK$Syy5 zbgfhDS-%*Y8mzrI>k$1G8s3vAq0-E`H5wd{VsH@!-&_w1Tc$ZF)!FAEB5v_2JV*Wj6-f3K)&t`IR8Qd-? zE0&vhy5EeuaTm*KHB1MCvZ#8Y1HV^69`l)LxuR5prBoz;ReBiD3%?Z z^;IwN+?*3HbN_gy63dM;+^~VIIRGY5i&A?;abnfBvJEczG(~w|pJ9kQH^ZMO=)+m* zwThl%sJ2+QJCKECBLg=P;ry$LPj9)dgs@8_g5{D#RE{5M zL52uX$%Bt%?fjPkW?Yi%wB8t&W=Z@D|XJK^Q% zt>`s&wy{dAg9LQ|NhyCcadB_XaG04}_G6*T>+>7!YCbG7-h5)%U9PJwrghmIov!T= zoR*5p9f#Z6nrVVCcOFM_nVk-Xp<=5@|5oT~=A~H9PXn>QX*`ki(}$+j(=&^yYkdn( zpYO?WrHWbOJ2cnvz`86W8uEku6yk%tq2h2d<(6K2f>eQEa8ZIZw=eIQ@of;ah{TQH z;^3O2=Qre|mw?l0z??AaQhqmC30Ti_7M3Yi}@U(|kXx0YLkZwUMm7Art;aiHraqzJ1=MGjS zou0~P^tedV+yq`Vqz&^Oc{@*gLY%7%@Iw7x>_cAvTCtnmPW*i7{OeoI$EEUfYj7Qg zM&5iu?|}de(tJNr$|+N5_**z!p{r`gaP$wZC23`)q>My!O#F1A>JLwA&9X1jzmuBb zY&tpz!zViPLONGYb41==RCaG{*NhOw!=Z5pTUTTEjZN`fL*;F{C-+@+ zF^S`S(xGsa2TSZ=4ll7jnYdK^2NkOM;wHo}L|uQ*)+2H|ExqF$g)Yn}^!kLYQ=Ohb zSn0*MosER>QQPwXY*z~u`F+NWcOA?8g`a=lhM0pTix_`w3e4Qv&qlu6e6E z-VrX*vGX2kM~st-&_6x}ixH!^tG9|}e_7C|(H$$uYAifR9KTAgGF|Ubf>Uf`D)O-~ zm(c%m7HaJ(tgoR_O7vV?JgFgxAWw%ARn)K%*Ha@|KIcO*_VwfF{D!+-16y!CYlVo_ zr0tt5UI0l@UP~6H*q(};a%JC5^>@{wF6jWC(>*B0COG^LxJs=Or`v|3}j;uC%^A`H|o ziH1hsnPmtwaddZ*=`6dRj(iSG3BlGI<7 zlHvH4#qD!B2eLRXi$mj5rg8pviZZ(aO#x~gdUx9FQ7VRpurxV|u8*iE`5R4V*^Z6x zq&_%4w0!?wSJ+r`h}1y2ded%|faa{*oq!Z|C>1 zm$&`nQNlNb2X<3BsE&~{X1=6|2s77k7?;&wgmyUXndV0g%O!@!r2M7TsC}FH$tr3$Nb1_S zSvrDM*GQxpYZm?L1TmL$PY7?vlcV%I119b1TP?b((g+n+~$&qzyi zpd(MWFb>}Oc2XPCv|6lGwOE@Ey`_-*&Of85wYsqsYh%u4L6?^7SC$}S7< z{(--7Pv796YwiOb%_^Pk)}QHGJh%|Qx@CP|=}6`&+4mLGafD~PfN*oo0X+6b!fpc( z@M55`w*}FV&ypQZgaWcZPY{Mqqw;vaOT!%PE*m;^<$_l2-$RnIBiiTWW~IX`nWpnv zC#Ihni(sX(gy#?}<7m(KaQylQcm11-io9wfC_mWi8g2ko=Cj4p`ANEFSYONa?AnYU zm`|v@xJnO{n!3^!`|yA&b*J z?y5a-2(on>W?$WQrY=pxzN~pfk6OIfl}X0`Bl4+@)*|=!p1G=7tyhgxxeBrE=8A9a zjGcg@r4BJ)MTi)4D)EZTcgAGAALPH!Z6PL(=qh9FRa-}=iYcpO-SOD1t?TCzq0j-_ zYhJV*w=gNM*Uv}kRhvI*KAOFpNwd4|5Nds{{=RrR=5PW%gI6j3D|?1qC#Art#G9jp z(p)Z&nUu=T(|a@VIN!#TknjKk637l2Ij}5G?=o zLNn#L6CfbUqJm2$`oq-qjfLZVfK))BqhWv4yRHE;zc-W(UA6VRqja|bo)Ct*XqgnxvY%p-X_vz(HBSeo$ME~uc+vmH zlx*=IN;ethA78~OmWX`mer7;vZQ0jHg^>%OPTY9+I>^=ZNV68{qtPL$@*~5;D!@@M zzMt~g35pxaCVI`|+i&4!6Xs+Htdg)QA%Z>yDXB2r@)XPXUr@k5OPkq$kc`Q8YiyuJ zpXMh*x^g8lC%TnUi;*$CUP}t(n-&nK+ww-8`hh!FhD+m~zLCilcDjNzZCF8xEuL-L z;L|0TrPN(~z*i#nHc*;ht27cl-lic4C8Cg@%Wp3qfS;?*E;;N zL7u6pA)*vBzNUm+Rxy4OGmQv=-Px+aLzAup;?&Jov_m=!)0!A*9I7_{Yv|}9`Xz&h zMeIcEj}4A9-^#emds3S!V1zPKG8P%zylYoKL}iJKuHqA$B%}WzXq);2CR?`6I`BpU zzGI4#VT40+^|AlL!`-xGQJ$vJaLqBz0e1)$Y!=ZqI78r^o%HKo67G&&;xA5`Djt#8 z5hTxxRml9PK7tAAlBku{3P+I8@Ic7d0<(PyIH4^}nwO$Y#Y|z@Iarvy&Ldn1 zr0D$JRY&tdNQ5v;dP%iX3)>)QSKwQWMQIn6yc+9qt$DzB?&>4(oC+dd>iiL?Wb|uh zw}P4NA&$@rv_v?v>yL0_458Ho^&dQ2n1P~r`?|&CW9CeVE+8;eYNHkaM5Q$Mx^-Yl zWX5r~-}@n=eqH%{?8p(%Iq9g7#?tH2f{CG%1}@oDtE#*n|>5V};L zZjsoPm7nAbcjUw_wC*q__n>@QfRVa<`d~Y9km2|kD1_x)0KzE7{mvE0eV~%XJqJ#A z)1GYl#596x@f$KAOzHNx2vy!4!~>Fo==Pv74lh;gwRV!x()AVU<_IJ7j|tFPiP?++ zYjqacWTuX_Hhb1!TA(A$!W1Y551orOBz1A!(Q}snuSKY8wF!wf!E?z!ryG$TiL!XH z_Mrq9VEioD{KYg(6N{yD^bfPO#jzOW1+Wl65sSfcm zLHj$C{pIm-={D{*7*5|`d1DGqptvu$a9K9CkX&eVnL=*3n%I^5kX!K;9SjGZq>2#> z6I0c?ufEoO#F9G$_6urXB7y%30|Te2n0H56Q6^+1ApfY1qXnsRUO1O-E)I-moiUP> z)iXmC0-p`tac6x*uYGPHnCxo3-($5#hV-}AX`3fYh20Rr5btB5B}OH}jc9%M-pc{; z_d%7Vd|YorBIu9s!(L;HkaXT}qMi+xYa~+I1Il#{_B=|4a}>AHZgED&bGzfF=N-V>9?L@AW=J`fUAHQ~(w)5(jvSUcSZ((2iq<2^-W)wVPs-3Fa= z2*07$g0L=p9aBv4i;erPCSgD`n`G)V@+;vPb{Iyi+r89q3M5zQ!t$fR5O>oW%tJeH zV%%}=IE3FP`$cFOKh!`BZ(H;a5v>d0G67z-i%uoCdW99RP0OZSF-+$@J|-jrqc{rlb^ z(eAU?7IHs%1wHC^x}ZS>>)TVbVuYR<-K#a(FydYHue?cpi^leLa@ayJee6S?nlA>G zlq1w?N1B_+>dGjVN?=UDi9`7Zbvi3O+pF%#uCYVnxOLD}Vx0W0x025_AEtO4B>5(# z*}D%k>z|m=Uda}2RN!_WI`hqzB&SP<5x2M!E2CCiEtTz1`=p<^0Lw(lFK|Oz5znRa zE!{Dp3)Q_bGHXO!x%Fs;(D5Ul;--a@-M*1gwdm;?)*FeR_i(?CgYQ8E#G3&C`Evvf zvBX4tc1!Fr{PTVOuMrlwIak!}8+73bhqe<1S+lkE-WvtAxldNIT%zopA#WK0+hCA} zq3xl)*`#|Hd=g`x__3-DQM7=mJ}$~%ECHj&uB$iDG>gO9Nae~qJ=R#d8jCr&d+s1) z!B{ABCgzhQohf;OHOEDTcxtztpJnDZM_RDMuntT6`Guo%|LTgG|MVN>i3>u*eK)~=RZ1DQI9VNoA4HH%+zW*KT9j84HPcCS3Rtf06t;**9V7CkKo*7lox^&>yU zK+H3z*Xghx=sT%{`ljv#rRi3(wd2M@l289nU^AvQihx!~t;=8jlU1qCRq>VO> z8zHBMgnB=jzbgKIV9%Zg84t%j7nc+)nkD*BDvPNTFtRbk`ihKR3&DL7jv^Gz)cHh> z59n^R<6HPrx1i{Pv@t^5Ii0dc)1 zhwz~z%0f}N<`fHEHcM-2MtLB%`Vr!L^7l2FSWnt)NEOKI${&cD&36r;rV!qAgWg4B zm`SDlSYkzh3w5u8zw1Sa$9$2ZP0*pJ@iX;J;s=;B&-JKF++m`gqw=uV>-kX*q;VYy zFk`_fkjUnb7G}&>aGWQqy$2Up?kW){Jm9GO8j`+DqE{n>G0NdK-H(_sa(p|U4|n6q z-D3sK?Y)KzsXLoiFA97T(WD`PO1wy`%s7JI-N+pjl#VHzge!WOaGd;Wq;+U({aS1U zS$)`9KO}FuY%&^*i6tguF2N1 zA75t$3Hd`5g+AQaaBptFU)G1 z$u)xTx#n~r@Ht;*&M`JId;Y|?cb_Y_#TDIvK!lsc?a^XDF2q_QBz(x$!EJlcErAm+ zS|Os5^fCGZo%{?b1f@tWoFxzeR?hIzVe!|AkAu7uK9>i#w~Fp7)t4s&7KmP?rkQ*w zvq`acV3;=o)&@=X>&Hd^=tHU$1YK&v$K^F4vxi@={3a8|4+JQz1r2^f=}c1RD4#vg z!D9+y9`~yf+vOSQd9^6FUE|b+=r-dEji(+1CrW`}&fYsVB0y-Q4^b>|tQ0~mBU&x^ z{e#3r_mn7hrrK`eUM6u6ljIlMG8u`iI%^GkZT794I>sv*AfZ`&xdJvx$z&o)te0*2 z%3LsM_kf_7-7ygK5LCyy`Io)g_8D_s;s0z&M9d9XYB@4KWWJ*O31Tt7r2 zFvY_|lH!By4UIbJC-1 zPL>OpnS_QW|5G9z6gNt|n8JLC&?!)l)F)@Dn`Ehyk(!nkE_ki$+P*GLN%<))i~|M* z){9)!|1Qy;hoD8#FvcnRfuzU*c4yN;o@i3iXjBTDyA_(F!9`4{sh<|{0$*= z(xLC3KEHS3=|pGEsB$7$n27!?5T#_}k?R2;qyz-y;s$g>rbJP1V(4G`Ec}+lGFu^T zn;b=S*CEFTI2OC%=U}suDb&(cGcSiNyw*F!u5AlY5Cz+cvMrzvuM0|hMCJ6@VF>g# z+j}Hji<#$1WFC6dL$c*lqbsFZx~(44J`qXMjI;^_(Kd52jgNPdFWuhPM_m({y4DMB zi~*><{c~c?IHEBNWWrs?%HSG~DX_a+>3R5Uq8+4i>nsM6s2;I7Zd@{lTusJ}kma`( zcf~W<9?%@8j;;>55pqvH8kQ5ACU*$Gmsg!Ka539FEu&X(d#~V$i{{!c1ZU1XKPDyh zcRsZl5&NBe;rXkv+~75;e6^O!F6h;9@tLbDVy8dbOo4N8*aOdPI7UZjGqZ?&M#mUZ zQx})_l&iQU9`SVm_?fx=NONRBwooUGJxPAg4w|D*){xlm`t-9_#$kc?ke6V-@&%Y}Qk7in`0o>3Nbo;Okb| zu>}w%aK`E~XL(JEurYm7N?%c^^DkXqw(1l)7iq51WmP`ub243<{kz>ysDx=2;O7j2 z%Lfl>#r6TqCWacDxib$kxCIuDClhjI+s-w`%tQMyG+?13iT8Z%{S^s&uR}~VY$l@X zQv}xtO9zL4JOdhbu5EA?H+;O7Ni+A%1ZU%;@UchNZe@GMF<1n#(hE6$5tnCWucK@a zax;uU&H!_^sn&V>6HnNnz^ijlZ$X&cofV>8GrUIYhm&egaD{nM6XoO%&#|bRUZpab zo#7;|bvHhX#-$dNOCh%JVBfco@OL7$#VIY2GBqrpuLMueurKX~9of-!Fw?$d)Hihm z?Y@3|*Bo^j;P?rNZ&Iy`c7%-{yQ`lyb*tbnI0%kKjoD|j*oE@CaR7CO{dQPfi}M^+ zh`?eo9-a6e3H1`((($ne`}@YCnj&2BX>J693_-1Vb0U;E&89cAdMZYwTzxVR9Y=rOl89>tz!s=TW$goJAKZ-JbPhT?5V6-Yry;fK zGY`g11}S;Vc8fp3f0?EmDRr4HptW%#pnk+ER%Am&*9qD7Ab@a}rA308&KvY#%B|@W zgTKk~x8FdpHC; Date: Fri, 6 Mar 2020 05:58:59 +0000 Subject: [PATCH 04/22] Readme formatting --- docs/google.md | 14 +++-- docs/images/google-cloud-logo.png | Bin 13226 -> 0 bytes docs/images/google-cloud-logo.svg | 96 ++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 5 deletions(-) delete mode 100644 docs/images/google-cloud-logo.png create mode 100644 docs/images/google-cloud-logo.svg diff --git a/docs/google.md b/docs/google.md index d2b0f51..d3a65f5 100644 --- a/docs/google.md +++ b/docs/google.md @@ -1,8 +1,11 @@ # nf-core/configs: Google Cloud Life Sciences Configuration -![Google Cloud](images/google-cloud-logo.png "https://cloud.google.com/life-sciences/docs/tutorials/nextflow") To be used with the `google` profile by specifying the `-profile google` when running nf-core pipelines. + +![Google Cloud](images/google-cloud-logo.svg "https://cloud.google.com/life-sciences/docs/tutorials/nextflow") + + ## Quick Start ``` GOOGLE_APPLICATION_CREDENTIALS=.json @@ -10,21 +13,22 @@ NXF_MODE=google nextflow run nf-core/rnaseq -profile google,test --google_bucket ``` -## Required Parameters +### Required Parameters #### `--google-bucket` The Google Cloud Storage bucket location to be used as a Nextflow work directory. Can also be specified with (`-w gs://your_bucket/work`). -## Optional Parameters +### Optional Parameters #### `--google-zone` The Google zone where the computation is executed in Compute Engine VMs. Multiple zones can be provided separating them by a comma. Default (`europe-west2-c`). #### `--google-preemptible` -Enables the usage of preemptible virtual machines with a retry error statergy for up to 5 retries. Default (`true`) +Enables the usage of preemptible virtual machines with a retry error statergy for up to 5 retries. Default (`true`). #### `--google-debug` Copies the /google debug directory from the VM to the task bucket directory. Useful for debugging. Default (`false`). -## Google Cloud Life Sciences Setup + +## Cloud Life Sciences Setup Please refer to the [Google Cloud](https://cloud.google.com/life-sciences/docs/tutorials/nextflow) and [Nextflow](https://www.nextflow.io/docs/latest/google.html#cloud-life-sciences) documentation which describe how to setup the Google Cloud environment. diff --git a/docs/images/google-cloud-logo.png b/docs/images/google-cloud-logo.png deleted file mode 100644 index 07c9a325b254d93f3fe4e94abb5cbdd9162afd92..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13226 zcmZ{LWmFtZ(C#em?hxEvg9ZyO!QI^@xVuYm4=%yoC1}tM9^5SicY=SL_ulWE`{$lL zXJ=-&d#0zlx}K`1s-u-ZNuweWAprn@Dk~$Q3IL!R$oVG(ILP;Q0N3;+U|0D$lOwNph1@&}x$ytD-H z_U}{B^DPAcIQwNK#MHf3P7S>M)HLqi@}81wmEYs5zsGzhZJ`2#uReZsc4)m;peQ4k zt$ExYPjv_$LiN<2e$S$qjdNeQ`-MDb_x&r*y}Q1su{kCNJ&Qw%=hS(EX4w048bw2` zjeD-YS?}w3@AEv*U-DefI(}maP%I%g?61#Amtzt^570xQ!a1`CiIJOg3y0+a>Oc<2 z9_~_#-Ct>IihAxhwVjBd@sA8alYuZqY+8jA=z_`Fq+2V}u3l9q{>Z4;p)Ew9Fl+pts3YF0?dq9pjU|kyJuiE?tw!fxd z<4?a^T7^t@lL6`*>IRdeChZ`AD4!w>skt1#KanwC-N zgjUBRs;j6=bl^qM>!d4l?3s78R^UKbnn5-Ag?%v6-c>g=74>ea~W)aCl zvE(Fkl6wu&QEbbBW06eYivJ^!PArL>q=&HG5zO5lR*3_^8Q@}B=Kg%L`M(YByu~-U zZK~geq=WbhkjEt+DyO0Zzpb^N%I0LKff-Ey6Kf|Fif4z@SE2VAnM^CQk52M_zkG4d zh}`LTqqt3nkKGBg0t3!3fV{kJF#9offfA!yC*J!|LzcW8P=Fq7h_i7%x~>AE*CT+ z3Ek#|&-TdVq#KNx&uaW@jHrdxVqUNP$1b>;MicC0=A*k8Is zuuCNptn(2W0+Hb4d$~%@2PELJ9;dJq-q}-3L;`omG2(wa?-ABMT-Ek4U(<(l2K-U{ zaJGq08~)Q$ehhrXYXHTYcK?*SWitk!FN|(UqkW~<%JwatD3}-yFV)W0 zJix0>K5r|h)mH-pq*=FDm~bbjsUp)z1Mca~7vWr%c%P~=jh<^eNE9=dkoVm1NQhMa z(yKm*h8#|WkxF#Sb8=vMU`>!|Z=Z`>Scqiqa3pR{3sI<|Rme%vjfJMwxeVa*?>~^*?%s)$tMC+Tub7o&?pAQI`76on5VATs@JpXOu+y6QW6*1h!s;8-GP}Zbdqi z#R4!1;wgl{K=9085h(SL^?e=mVI*Mcm!^|pcD%;{whw{bST4jA&?Q~rTEFamuH^QP0~xKlIfahLhN2e1ly28=O- zx&jG#YriSfh>%HWVX?no+4NsH_geE*s2cOT&(3j^cHf`P@nJnj!=|!|c!H6+)Y)DcNtGL|zy{vWL0wT!e06CSk7=3vtKB-I7{^eJ^-b?Y)}=_ny?VeXr-; zYN*DbAXs9$ge(Q{Jv8?KIj2+h^-a(zA*q2oM}j7v9 zmreoz$Cy5!()F}J-_tKO=;|686=ls`B3P6+MorSUXwXe4r--|Xc+RBld!c0UwpV>L zWX#(qCo?&zD|U!}Pv2gKb`9hHvxJU*98M+F5XqfCzMBdL+jIJeRl0`iaoX- zHjSh~0zaO<)gr$UI~_tiy`tX%vXOU{L)lhhY5pd;$$>F1($)DMzT>SxE zYfm3m?oIJ~$*PvYRB#kDNzWmY4o&Ju+X9kyc zc7PvV_vbZ(2?T8~Q83AHrG2EqZMr4Pc{bz2=9!&MyAsHN_H$N@`l1vSF;2 zcKmom)+R=%4BdD*#C5wdSKP@0+xH7oyAG*07+v()R-#2HJ-}zZvX$h*cX771V&mwV zO4AQm=UQ)fUI|iUG|P~(P%b4nk1BGJ~LbMY8@8z#uHh5qP%n}&fcGDEwpdA|VP z*Vi#*k5EQ9J3Fkv#$a2rOTYHJLObuU*)7;D7?btROIcIXGWp+NzRfMC;QHXMEOWPU z?oJg3onMViOr+4RCid?As!xl+HdARzSYS|Cwk)JhnFaw#2&|Y{CgXfos6P9Fp{ba9 z*fgJWUoZFp7BsdNRvT^0cgvd4MtApl69A)P>pNP3UZ(Ct1m;C@DIY^q91S?R^vnr} zN3dHEGcOI0)HWPxGAPh#!TG&$UJ2&YGd%G8pqzr;zAQ7U(aapqxy4ox-;4)e2M7op z$JE)XMLNN*bY2jcVafX87@GP=WsLZ?RJzxPmEw!KC%=#g|C!9@$bF#cT0b)cYU4P* zScQT%{aj@sKu6+YsNm{Dp! zG4&l-;jwAsLLoR#5#6C{_bHig5CPwpSrz@kKRnIC!VdgT-@I&Awq$Z|p`lp+^aYU8 z^qq%gXwwp}*E6PaQU|8zjrD~@d(Pu{4tM=Q8J~Ijir~t`prAu)-5t=)~q`_xS(DURutWlD%7@7T3UkRpLFUxEoVyJJR`$C(ZCC*(lCPO9EHq2=A@)w zPLYEe^@)T^Bm#$hS}R4OaX&59W_@ozL{ck`te{%SQ21cHX?~2&ZtrTSOgD~L|Fb+4 zhL4C4{`}o@MO>-y&X#!YuIv?w?^-Fy>%V-vl3 z9JGlMsHdu05k3NqZPr?)xg@AWx-tN>7B zuaaOGw9e(j_TBJM(n8oQ4-f%VqZW@rka7Wg(7P5aNFOagB;+qkMGQU~vEP^!sH=SebKZ2=Q0B7I~_`6A_H%o zF%icm)bMos@lR|ta%0+{EHthDC-UzNJh6J4X4`~Nu;`r5r1Z5h(S*kO3e>EvT|ycq zF;yY`JE$;^@O>&wx8Z{opk|fu+O;gM1acd1ge|U}bXsHTx9sB^FmB`7nKAp4 zlx+N=YCO6j;?KaF3IE_xS4*5YeDO)mel=pPqyz3_Xk+e7Ddc{wI^C{qc*tbEs#8qp z`L<}#yi$zQ-oFc?!g@FI#URN{^x5QkGZsz=aqDVGKYAOB2FGS-)5 zjI(0!kdMJ$rNdcJek|8KZz-B2BZrW^$SeTM87>j567R6ec5MepZAO20oIU-O;nrS) z;G^|6?`$xhvu**M^}dFFV*3gP2lQ@UaYEUCPd zUOKYAf{Ul5=5BA@hIPhULh-OV3#ZX*yZn7;H|mJ}Oh9(FD0XD9iar@;KK~Wcr&Bb} z&wL~{Xc;4msdt@ezS!}m)8dg6*v-qevhV=q3Mqb}bh>UW=u#(FPw2U*kw@QJ-Hz#+ zA#RvEX8107ROGVNA<{iH74o!c8q5K$1QB5;>3ePVhe*BI0VcwF;-wRt>Wh(%$&j!R zj?>y_S;z%|vX*M?)5EAKkP90qBlF2SPf!#mhAJbldfA_z=|(uH`c7Txbn;mm;f6j) z9eRaWvGVWqA22WX=8uHq_oQ>??(Z{}TUXxYY>M(mtnU-j4g7@(i$!6V`eJ_YPhi!x z%qbJdNRRc&tzUzmVQ>#=OG)qIstxKo*3VEZC1`GX^v|B99t#jC+dLdEr>oKl#bib-B-$KM^lOt_*z zPhe7g{uFXu!;9H_Fgnr8t!?<|w>I>LkuSer30lSm`ZGxxR&P#@7JFf1_g>RNB0rZU z@lt~DsRAB4bo;=T<-Jxko}wpi-c@GrQsS+s;xc!!^#OAt`A$wYI?VGQf91R-2fkMbC=uwNxAeTEIfN^)I9eg( zLV}kY{o(NOpv%OfjP9iNxq>3_Pp=uaReuHeS)DlP!Rb&Q9BT=tCm~kqOBl>DGH1_f ztAiiTG8$?~rsB!x6otmnQmur>9nElYaTzAS9GglUq>rJHf_htYV7cxNnK4%XL4M6+ z9FPU8i4YYZ--a=KsZFK6#wlJuDbAT<~aD8&5$ddY+CDKS^7>BY-&2vO+2 zVP4VzrZj3tY~g+atH&Q4!WXN&iz)ww544>kXoQl0;c!6)>|fushAeWQls;wl{PjH> zIh%9ObLaL*u9=?CLXRJU9!f?tYy)0yM z?!oEQINgEj&mY;Frs_)?gl|l#B6q(0%p1DsNFQ>rvl-t4;D$ZZ)JZ+qLkFP)II9ME zpMe7qpXRrkPze}UkhKq`MCLjC;TFNl7(9kL#!W_b+=fv%>ei>OIkV6{{rF0q-t7`G zy9Fayi7A-_kFJ_q#s`r0Z3Y1z=5-4Mz>I8!RNt>_@F|E2ONe$2FV$0$Mi-JxM%j48 zNcz|Fo4|zrAO^Zowk*0PxX|K$ixKVT2NQOD^8Asvc!d=rdT#xNgQEVsV6B2ILmOy1 z2KSM-Vkf!mz981vdn~Zq@VugG!Lgfo%bGU~hU-~UQc~LeuIu<8sbJ}yFg%hIeK>WA z(|`5!q6gB?%E@?x3(kvk|4I=wEchG9wJH~b?e6b0(l(6?X9;DI`%8|;mqh5u&f++) zykmRR(MszIC`UV)mt99+@B(je#@5N0W9_x;_(TIO9L750ppyxO{yWf##?>hi&4l`4%xQZ)(u7r`&fR zN6BIa@+*HLihlL>789GlJ`qT>gmQC3aliX++~-+_mIV!vUgHNE{khNm*!S`DcEdFn z)(VPG@pqB{gYzeLWf!MvrcV^T=kHX9_yv5?F{6D^mR?kg&p{+&yn}99+w5@7P^p^} z2B!23brVF6oP*xLeN*q=H)>H>S(QKTTSNa%hIozhpgP4EuSe8EP&Hs>a z3q2Ds?RGird#zp2#t-hJ64ApDepjF&kYJ&6q-VI>yljsYY|4;K^Y3jj1M4y2U%pqF zF$%+p1-%tj{RVO2f$v#j0x2ac`#bv!==KHHzfpJH6iu1TltOjx1P>hjQt1(iQ^btX z|5{G-!CdrB55yeDC@r?QV&9rN$(TaBFL2~QY?+7q)#zD_@>CN}4iD=vI5P~bXabcG zagHUuZ9q+}EJ7su)_P^=(WxedcoOy+H7ZZOXS}ExJxh9Uc3J1mz(cJJHEsfQ!ddh) zN_o<0iUWnHc@r+taPv=8ZR2P5^AFOQ-Vu|#sOCxd)Gjo2m6dW1%6p~8hZaEsY`pg| z0lu#T`!2VzZf430Wx9Je5}d{lFRwM?mH2-)-rv-*1gLkJP5eUSzze^g z@iL#W9G#dDOhok2g8K+TqYCX)bKf4kt~pne+PS7wjZ0hv=_96uol5F1QuisoXn=pN|cJ1qW+Y)2g+XwG}TzvI)C2z=f81vbJImI>v~wZtpK$Syy5 zbgfhDS-%*Y8mzrI>k$1G8s3vAq0-E`H5wd{VsH@!-&_w1Tc$ZF)!FAEB5v_2JV*Wj6-f3K)&t`IR8Qd-? zE0&vhy5EeuaTm*KHB1MCvZ#8Y1HV^69`l)LxuR5prBoz;ReBiD3%?Z z^;IwN+?*3HbN_gy63dM;+^~VIIRGY5i&A?;abnfBvJEczG(~w|pJ9kQH^ZMO=)+m* zwThl%sJ2+QJCKECBLg=P;ry$LPj9)dgs@8_g5{D#RE{5M zL52uX$%Bt%?fjPkW?Yi%wB8t&W=Z@D|XJK^Q% zt>`s&wy{dAg9LQ|NhyCcadB_XaG04}_G6*T>+>7!YCbG7-h5)%U9PJwrghmIov!T= zoR*5p9f#Z6nrVVCcOFM_nVk-Xp<=5@|5oT~=A~H9PXn>QX*`ki(}$+j(=&^yYkdn( zpYO?WrHWbOJ2cnvz`86W8uEku6yk%tq2h2d<(6K2f>eQEa8ZIZw=eIQ@of;ah{TQH z;^3O2=Qre|mw?l0z??AaQhqmC30Ti_7M3Yi}@U(|kXx0YLkZwUMm7Art;aiHraqzJ1=MGjS zou0~P^tedV+yq`Vqz&^Oc{@*gLY%7%@Iw7x>_cAvTCtnmPW*i7{OeoI$EEUfYj7Qg zM&5iu?|}de(tJNr$|+N5_**z!p{r`gaP$wZC23`)q>My!O#F1A>JLwA&9X1jzmuBb zY&tpz!zViPLONGYb41==RCaG{*NhOw!=Z5pTUTTEjZN`fL*;F{C-+@+ zF^S`S(xGsa2TSZ=4ll7jnYdK^2NkOM;wHo}L|uQ*)+2H|ExqF$g)Yn}^!kLYQ=Ohb zSn0*MosER>QQPwXY*z~u`F+NWcOA?8g`a=lhM0pTix_`w3e4Qv&qlu6e6E z-VrX*vGX2kM~st-&_6x}ixH!^tG9|}e_7C|(H$$uYAifR9KTAgGF|Ubf>Uf`D)O-~ zm(c%m7HaJ(tgoR_O7vV?JgFgxAWw%ARn)K%*Ha@|KIcO*_VwfF{D!+-16y!CYlVo_ zr0tt5UI0l@UP~6H*q(};a%JC5^>@{wF6jWC(>*B0COG^LxJs=Or`v|3}j;uC%^A`H|o ziH1hsnPmtwaddZ*=`6dRj(iSG3BlGI<7 zlHvH4#qD!B2eLRXi$mj5rg8pviZZ(aO#x~gdUx9FQ7VRpurxV|u8*iE`5R4V*^Z6x zq&_%4w0!?wSJ+r`h}1y2ded%|faa{*oq!Z|C>1 zm$&`nQNlNb2X<3BsE&~{X1=6|2s77k7?;&wgmyUXndV0g%O!@!r2M7TsC}FH$tr3$Nb1_S zSvrDM*GQxpYZm?L1TmL$PY7?vlcV%I119b1TP?b((g+n+~$&qzyi zpd(MWFb>}Oc2XPCv|6lGwOE@Ey`_-*&Of85wYsqsYh%u4L6?^7SC$}S7< z{(--7Pv796YwiOb%_^Pk)}QHGJh%|Qx@CP|=}6`&+4mLGafD~PfN*oo0X+6b!fpc( z@M55`w*}FV&ypQZgaWcZPY{Mqqw;vaOT!%PE*m;^<$_l2-$RnIBiiTWW~IX`nWpnv zC#Ihni(sX(gy#?}<7m(KaQylQcm11-io9wfC_mWi8g2ko=Cj4p`ANEFSYONa?AnYU zm`|v@xJnO{n!3^!`|yA&b*J z?y5a-2(on>W?$WQrY=pxzN~pfk6OIfl}X0`Bl4+@)*|=!p1G=7tyhgxxeBrE=8A9a zjGcg@r4BJ)MTi)4D)EZTcgAGAALPH!Z6PL(=qh9FRa-}=iYcpO-SOD1t?TCzq0j-_ zYhJV*w=gNM*Uv}kRhvI*KAOFpNwd4|5Nds{{=RrR=5PW%gI6j3D|?1qC#Art#G9jp z(p)Z&nUu=T(|a@VIN!#TknjKk637l2Ij}5G?=o zLNn#L6CfbUqJm2$`oq-qjfLZVfK))BqhWv4yRHE;zc-W(UA6VRqja|bo)Ct*XqgnxvY%p-X_vz(HBSeo$ME~uc+vmH zlx*=IN;ethA78~OmWX`mer7;vZQ0jHg^>%OPTY9+I>^=ZNV68{qtPL$@*~5;D!@@M zzMt~g35pxaCVI`|+i&4!6Xs+Htdg)QA%Z>yDXB2r@)XPXUr@k5OPkq$kc`Q8YiyuJ zpXMh*x^g8lC%TnUi;*$CUP}t(n-&nK+ww-8`hh!FhD+m~zLCilcDjNzZCF8xEuL-L z;L|0TrPN(~z*i#nHc*;ht27cl-lic4C8Cg@%Wp3qfS;?*E;;N zL7u6pA)*vBzNUm+Rxy4OGmQv=-Px+aLzAup;?&Jov_m=!)0!A*9I7_{Yv|}9`Xz&h zMeIcEj}4A9-^#emds3S!V1zPKG8P%zylYoKL}iJKuHqA$B%}WzXq);2CR?`6I`BpU zzGI4#VT40+^|AlL!`-xGQJ$vJaLqBz0e1)$Y!=ZqI78r^o%HKo67G&&;xA5`Djt#8 z5hTxxRml9PK7tAAlBku{3P+I8@Ic7d0<(PyIH4^}nwO$Y#Y|z@Iarvy&Ldn1 zr0D$JRY&tdNQ5v;dP%iX3)>)QSKwQWMQIn6yc+9qt$DzB?&>4(oC+dd>iiL?Wb|uh zw}P4NA&$@rv_v?v>yL0_458Ho^&dQ2n1P~r`?|&CW9CeVE+8;eYNHkaM5Q$Mx^-Yl zWX5r~-}@n=eqH%{?8p(%Iq9g7#?tH2f{CG%1}@oDtE#*n|>5V};L zZjsoPm7nAbcjUw_wC*q__n>@QfRVa<`d~Y9km2|kD1_x)0KzE7{mvE0eV~%XJqJ#A z)1GYl#596x@f$KAOzHNx2vy!4!~>Fo==Pv74lh;gwRV!x()AVU<_IJ7j|tFPiP?++ zYjqacWTuX_Hhb1!TA(A$!W1Y551orOBz1A!(Q}snuSKY8wF!wf!E?z!ryG$TiL!XH z_Mrq9VEioD{KYg(6N{yD^bfPO#jzOW1+Wl65sSfcm zLHj$C{pIm-={D{*7*5|`d1DGqptvu$a9K9CkX&eVnL=*3n%I^5kX!K;9SjGZq>2#> z6I0c?ufEoO#F9G$_6urXB7y%30|Te2n0H56Q6^+1ApfY1qXnsRUO1O-E)I-moiUP> z)iXmC0-p`tac6x*uYGPHnCxo3-($5#hV-}AX`3fYh20Rr5btB5B}OH}jc9%M-pc{; z_d%7Vd|YorBIu9s!(L;HkaXT}qMi+xYa~+I1Il#{_B=|4a}>AHZgED&bGzfF=N-V>9?L@AW=J`fUAHQ~(w)5(jvSUcSZ((2iq<2^-W)wVPs-3Fa= z2*07$g0L=p9aBv4i;erPCSgD`n`G)V@+;vPb{Iyi+r89q3M5zQ!t$fR5O>oW%tJeH zV%%}=IE3FP`$cFOKh!`BZ(H;a5v>d0G67z-i%uoCdW99RP0OZSF-+$@J|-jrqc{rlb^ z(eAU?7IHs%1wHC^x}ZS>>)TVbVuYR<-K#a(FydYHue?cpi^leLa@ayJee6S?nlA>G zlq1w?N1B_+>dGjVN?=UDi9`7Zbvi3O+pF%#uCYVnxOLD}Vx0W0x025_AEtO4B>5(# z*}D%k>z|m=Uda}2RN!_WI`hqzB&SP<5x2M!E2CCiEtTz1`=p<^0Lw(lFK|Oz5znRa zE!{Dp3)Q_bGHXO!x%Fs;(D5Ul;--a@-M*1gwdm;?)*FeR_i(?CgYQ8E#G3&C`Evvf zvBX4tc1!Fr{PTVOuMrlwIak!}8+73bhqe<1S+lkE-WvtAxldNIT%zopA#WK0+hCA} zq3xl)*`#|Hd=g`x__3-DQM7=mJ}$~%ECHj&uB$iDG>gO9Nae~qJ=R#d8jCr&d+s1) z!B{ABCgzhQohf;OHOEDTcxtztpJnDZM_RDMuntT6`Guo%|LTgG|MVN>i3>u*eK)~=RZ1DQI9VNoA4HH%+zW*KT9j84HPcCS3Rtf06t;**9V7CkKo*7lox^&>yU zK+H3z*Xghx=sT%{`ljv#rRi3(wd2M@l289nU^AvQihx!~t;=8jlU1qCRq>VO> z8zHBMgnB=jzbgKIV9%Zg84t%j7nc+)nkD*BDvPNTFtRbk`ihKR3&DL7jv^Gz)cHh> z59n^R<6HPrx1i{Pv@t^5Ii0dc)1 zhwz~z%0f}N<`fHEHcM-2MtLB%`Vr!L^7l2FSWnt)NEOKI${&cD&36r;rV!qAgWg4B zm`SDlSYkzh3w5u8zw1Sa$9$2ZP0*pJ@iX;J;s=;B&-JKF++m`gqw=uV>-kX*q;VYy zFk`_fkjUnb7G}&>aGWQqy$2Up?kW){Jm9GO8j`+DqE{n>G0NdK-H(_sa(p|U4|n6q z-D3sK?Y)KzsXLoiFA97T(WD`PO1wy`%s7JI-N+pjl#VHzge!WOaGd;Wq;+U({aS1U zS$)`9KO}FuY%&^*i6tguF2N1 zA75t$3Hd`5g+AQaaBptFU)G1 z$u)xTx#n~r@Ht;*&M`JId;Y|?cb_Y_#TDIvK!lsc?a^XDF2q_QBz(x$!EJlcErAm+ zS|Os5^fCGZo%{?b1f@tWoFxzeR?hIzVe!|AkAu7uK9>i#w~Fp7)t4s&7KmP?rkQ*w zvq`acV3;=o)&@=X>&Hd^=tHU$1YK&v$K^F4vxi@={3a8|4+JQz1r2^f=}c1RD4#vg z!D9+y9`~yf+vOSQd9^6FUE|b+=r-dEji(+1CrW`}&fYsVB0y-Q4^b>|tQ0~mBU&x^ z{e#3r_mn7hrrK`eUM6u6ljIlMG8u`iI%^GkZT794I>sv*AfZ`&xdJvx$z&o)te0*2 z%3LsM_kf_7-7ygK5LCyy`Io)g_8D_s;s0z&M9d9XYB@4KWWJ*O31Tt7r2 zFvY_|lH!By4UIbJC-1 zPL>OpnS_QW|5G9z6gNt|n8JLC&?!)l)F)@Dn`Ehyk(!nkE_ki$+P*GLN%<))i~|M* z){9)!|1Qy;hoD8#FvcnRfuzU*c4yN;o@i3iXjBTDyA_(F!9`4{sh<|{0$*= z(xLC3KEHS3=|pGEsB$7$n27!?5T#_}k?R2;qyz-y;s$g>rbJP1V(4G`Ec}+lGFu^T zn;b=S*CEFTI2OC%=U}suDb&(cGcSiNyw*F!u5AlY5Cz+cvMrzvuM0|hMCJ6@VF>g# z+j}Hji<#$1WFC6dL$c*lqbsFZx~(44J`qXMjI;^_(Kd52jgNPdFWuhPM_m({y4DMB zi~*><{c~c?IHEBNWWrs?%HSG~DX_a+>3R5Uq8+4i>nsM6s2;I7Zd@{lTusJ}kma`( zcf~W<9?%@8j;;>55pqvH8kQ5ACU*$Gmsg!Ka539FEu&X(d#~V$i{{!c1ZU1XKPDyh zcRsZl5&NBe;rXkv+~75;e6^O!F6h;9@tLbDVy8dbOo4N8*aOdPI7UZjGqZ?&M#mUZ zQx})_l&iQU9`SVm_?fx=NONRBwooUGJxPAg4w|D*){xlm`t-9_#$kc?ke6V-@&%Y}Qk7in`0o>3Nbo;Okb| zu>}w%aK`E~XL(JEurYm7N?%c^^DkXqw(1l)7iq51WmP`ub243<{kz>ysDx=2;O7j2 z%Lfl>#r6TqCWacDxib$kxCIuDClhjI+s-w`%tQMyG+?13iT8Z%{S^s&uR}~VY$l@X zQv}xtO9zL4JOdhbu5EA?H+;O7Ni+A%1ZU%;@UchNZe@GMF<1n#(hE6$5tnCWucK@a zax;uU&H!_^sn&V>6HnNnz^ijlZ$X&cofV>8GrUIYhm&egaD{nM6XoO%&#|bRUZpab zo#7;|bvHhX#-$dNOCh%JVBfco@OL7$#VIY2GBqrpuLMueurKX~9of-!Fw?$d)Hihm z?Y@3|*Bo^j;P?rNZ&Iy`c7%-{yQ`lyb*tbnI0%kKjoD|j*oE@CaR7CO{dQPfi}M^+ zh`?eo9-a6e3H1`((($ne`}@YCnj&2BX>J693_-1Vb0U;E&89cAdMZYwTzxVR9Y=rOl89>tz!s=TW$goJAKZ-JbPhT?5V6-Yry;fK zGY`g11}S;Vc8fp3f0?EmDRr4HptW%#pnk+ER%Am&*9qD7Ab@a}rA308&KvY#%B|@W zgTKk~x8FdpHC; + + + + + + + + + +]> + + + + + + + + +Cloud_Logo_Nav + + + + + + + + + + + + + + + + + + + + + From 44cd3089def3f47cc1a53a6f7536c4d203e05fab Mon Sep 17 00:00:00 2001 From: Evan Floden Date: Fri, 6 Mar 2020 09:48:00 +0100 Subject: [PATCH 05/22] Alphabetical ordering of GitHub workflows --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d68f44f..5e9676d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'google', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 8e42bbd7d055acae8fff3ed8fca372979a96adfa Mon Sep 17 00:00:00 2001 From: mseybold Date: Fri, 6 Mar 2020 10:25:08 +0100 Subject: [PATCH 06/22] Update cfc.config We just added some new HighMem nodes to the cfc, Partition/queue name is qbic, each node has 128Cores and 2TB of RAM. The old queue (Name compute) is set to default by slurm. Each process over 60GB of RAM or over 20cpus should run on those new nodes. So added the queue State and adjusted the max_memory and max_cpu setting. --- conf/cfc.config | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conf/cfc.config b/conf/cfc.config index 763dcf0..99917ba 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -13,6 +13,7 @@ singularity { process { beforeScript = 'module load devel/singularity/3.4.2' executor = 'slurm' + queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' } } weblog{ @@ -22,7 +23,7 @@ weblog{ params { igenomes_base = '/nfsmounts/igenomes' - max_memory = 498.GB - max_cpus = 20 + max_memory = 1999.GB + max_cpus = 128 max_time = 140.h } From 52842de88b0d64e2bdbfdf61a833b975879c2546 Mon Sep 17 00:00:00 2001 From: mseybold Date: Fri, 6 Mar 2020 10:46:37 +0100 Subject: [PATCH 07/22] Update cfc.config test failed, so added the the default queue --- conf/cfc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index 99917ba..d1a109a 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -13,7 +13,7 @@ singularity { process { beforeScript = 'module load devel/singularity/3.4.2' executor = 'slurm' - queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' } + queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' : 'compute' } } weblog{ From 2135996b13220dc76b4fe1b1196384307796c317 Mon Sep 17 00:00:00 2001 From: mseybold Date: Fri, 6 Mar 2020 11:12:21 +0100 Subject: [PATCH 08/22] Update cfc.md --- docs/cfc.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/cfc.md b/docs/cfc.md index a7f6beb..acf93dd 100644 --- a/docs/cfc.md +++ b/docs/cfc.md @@ -10,8 +10,11 @@ Before running the pipeline you will need to load Nextflow and Singularity using ## Load Nextflow and Singularity environment modules module purge module load devel/java_jdk/1.8.0u121 -module load qbic/singularity_slurm/3.0.3 +module load devel/singularity/3.4.2 ``` >NB: You will need an account to use the HPC cluster CFC in order to run the pipeline. If in doubt contact IT. >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. +New HighMem nodes are available on the cfc, Partition/queue name is qbic, each node has 128Cores and 2TB of RAM. +The old queue (Name compute) is set to default by slurm. +Each process over 60GB of RAM or over 20cpus should run on those new nodes. From 48400b8863b11dc2e70917fa3d586e56e5e41f6f Mon Sep 17 00:00:00 2001 From: mseybold Date: Fri, 6 Mar 2020 11:14:27 +0100 Subject: [PATCH 09/22] Update RADME.md fixed link to cfc.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0e58b09..3b360a1 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Currently documentation is available for the following systems: * [CCGA](docs/ccga.md) * [CCGA_DX](docs/ccga_dx.md) * [CCGA_MED](docs/ccga_med.md) -* [CFC](docs/binac.md) +* [CFC](docs/cfc.md) * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) From f5364e45e7c007d9da5b24dcb50c5329b53b36ae Mon Sep 17 00:00:00 2001 From: Evan Floden Date: Fri, 6 Mar 2020 11:27:44 +0100 Subject: [PATCH 10/22] Update google.md or markdown linting --- docs/google.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/google.md b/docs/google.md index d3a65f5..0534a0b 100644 --- a/docs/google.md +++ b/docs/google.md @@ -2,33 +2,36 @@ To be used with the `google` profile by specifying the `-profile google` when running nf-core pipelines. - ![Google Cloud](images/google-cloud-logo.svg "https://cloud.google.com/life-sciences/docs/tutorials/nextflow") - ## Quick Start -``` + +```bash GOOGLE_APPLICATION_CREDENTIALS=.json NXF_MODE=google nextflow run nf-core/rnaseq -profile google,test --google_bucket ``` ### Required Parameters + #### `--google-bucket` + The Google Cloud Storage bucket location to be used as a Nextflow work directory. Can also be specified with (`-w gs://your_bucket/work`). ### Optional Parameters + #### `--google-zone` + The Google zone where the computation is executed in Compute Engine VMs. Multiple zones can be provided separating them by a comma. Default (`europe-west2-c`). #### `--google-preemptible` + Enables the usage of preemptible virtual machines with a retry error statergy for up to 5 retries. Default (`true`). #### `--google-debug` + Copies the /google debug directory from the VM to the task bucket directory. Useful for debugging. Default (`false`). - ## Cloud Life Sciences Setup + Please refer to the [Google Cloud](https://cloud.google.com/life-sciences/docs/tutorials/nextflow) and [Nextflow](https://www.nextflow.io/docs/latest/google.html#cloud-life-sciences) documentation which describe how to setup the Google Cloud environment. - - From ebae81ceb3a5a9d07d72d31b33867a369944d889 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 6 Mar 2020 17:18:03 +0100 Subject: [PATCH 11/22] Update main.yml Drop CCGA, as it was removed in an intermediate merge. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5e9676d..36f9da1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 6141e008a3f082bb048aa5f41743bcc4bc706270 Mon Sep 17 00:00:00 2001 From: Evan Floden Date: Mon, 9 Mar 2020 14:44:58 +0100 Subject: [PATCH 12/22] Update google.md --- docs/google.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/google.md b/docs/google.md index 0534a0b..9e8c521 100644 --- a/docs/google.md +++ b/docs/google.md @@ -9,7 +9,7 @@ To be used with the `google` profile by specifying the `-profile google` when ru ```bash GOOGLE_APPLICATION_CREDENTIALS=.json NXF_MODE=google -nextflow run nf-core/rnaseq -profile google,test --google_bucket +nextflow run nf-core/rnaseq -profile test,google --google_bucket ``` ### Required Parameters From de914e56eed0a86d739f3f5fe74897858252aa69 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:08:01 +0200 Subject: [PATCH 13/22] Create ebc.config --- conf/ebc.config | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 conf/ebc.config diff --git a/conf/ebc.config b/conf/ebc.config new file mode 100644 index 0000000..ebcb775 --- /dev/null +++ b/conf/ebc.config @@ -0,0 +1,26 @@ + //Profile config names for nf-core/configs + params { + config_profile_description = 'Generic Estonian Biocentre profile provided by nf-core/configs.' + config_profile_contact = 'Marcel Keller (@marcel-keller)' + config_profile_url = 'https://genomics.ut.ee/en/about-us/estonian-biocentre' + } + + cleanup = true + + conda { + cacheDir = '/ebc_data/nf-core/conda' + + } + process { + executor = 'slurm' + conda = "$baseDir/environment.yml" + + } + executor { + queueSize = 16 + } + params { + max_memory = 12.GB + max_cpus = 20 + max_time = 120.h + } From b8d020b78c54b162f5178f531fcc4dad9c69f5b5 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 9 Mar 2020 15:12:49 +0100 Subject: [PATCH 14/22] Update cfc.md --- docs/cfc.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/cfc.md b/docs/cfc.md index acf93dd..f96a47d 100644 --- a/docs/cfc.md +++ b/docs/cfc.md @@ -15,6 +15,5 @@ module load devel/singularity/3.4.2 >NB: You will need an account to use the HPC cluster CFC in order to run the pipeline. If in doubt contact IT. >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. -New HighMem nodes are available on the cfc, Partition/queue name is qbic, each node has 128Cores and 2TB of RAM. -The old queue (Name compute) is set to default by slurm. -Each process over 60GB of RAM or over 20cpus should run on those new nodes. + +The queues are set to be `qbic` or `compute` and will be chosen automatically for you depending on your job submission. From 418f4b52b66125566aff9e985bc6ef9b30229875 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:23:16 +0200 Subject: [PATCH 15/22] Create ebc.md --- docs/ebc.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/ebc.md diff --git a/docs/ebc.md b/docs/ebc.md new file mode 100644 index 0000000..f997f9a --- /dev/null +++ b/docs/ebc.md @@ -0,0 +1,8 @@ +# nf-core/configs: EBC Configuration +All nf-core pipelines have been successfully configured for use on the [Estonian Biocentre (EBC)](https://genomics.ut.ee/en/about-us/estonian-biocentre) cluster at the [High Performance Computing Center](https://hpc.ut.ee/en) of the the University of Tartu. +To use, run the pipeline with `-profile ebc`. This will download and launch the [`ebc.config`](../conf/ebc.config) which has been pre-configured with a setup suitable for the EBC cluster. Using this profile, currently, a conda environment containing all of the required software will be downloaded and stored in a central location. + +The profile will put a maximum job limit of 12 GB, 20 CPUs and a maximum wall time of 120 hours. + +NB: You will need an account to use the HPC cluster on EBC cluster in order to run the pipeline. If in doubt contact IT. +NB: Nextflow will need to submit the jobs via the SLURM scheduler to the HPC cluster and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. From f48a4cbfb8f4a3ad8845d2db7bf13759bb4c5888 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:25:19 +0200 Subject: [PATCH 16/22] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 36f9da1..042dd26 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 88cfe1118738067c8d47ecb567e5d4e0b0184cc1 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:32:14 +0200 Subject: [PATCH 17/22] Update ebc.md --- docs/ebc.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ebc.md b/docs/ebc.md index f997f9a..45c9415 100644 --- a/docs/ebc.md +++ b/docs/ebc.md @@ -1,4 +1,5 @@ # nf-core/configs: EBC Configuration + All nf-core pipelines have been successfully configured for use on the [Estonian Biocentre (EBC)](https://genomics.ut.ee/en/about-us/estonian-biocentre) cluster at the [High Performance Computing Center](https://hpc.ut.ee/en) of the the University of Tartu. To use, run the pipeline with `-profile ebc`. This will download and launch the [`ebc.config`](../conf/ebc.config) which has been pre-configured with a setup suitable for the EBC cluster. Using this profile, currently, a conda environment containing all of the required software will be downloaded and stored in a central location. From edc6a6aa87946ba943141298a7b262eb74aed754 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:35:45 +0200 Subject: [PATCH 18/22] Update nfcore_custom.config --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index d7f3ca4..848db80 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -22,6 +22,7 @@ profiles { czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config"} + ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" } google { includeConfig "${params.custom_config_base}/conf/google.config" } denbi_qbic { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" } From 3b575195009b748cf632b4cd5294d32e3edc031a Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:48:50 +0200 Subject: [PATCH 19/22] Update ebc.config --- conf/ebc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/ebc.config b/conf/ebc.config index ebcb775..4b1332d 100644 --- a/conf/ebc.config +++ b/conf/ebc.config @@ -14,7 +14,7 @@ process { executor = 'slurm' conda = "$baseDir/environment.yml" - + beforeScript = 'module load nextflow' } executor { queueSize = 16 From c00bbe982b7250898c9310e8bb9fbb6e774f1e13 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:51:38 +0200 Subject: [PATCH 20/22] Update ebc.config --- conf/ebc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/ebc.config b/conf/ebc.config index 4b1332d..ea48f05 100644 --- a/conf/ebc.config +++ b/conf/ebc.config @@ -9,12 +9,12 @@ conda { cacheDir = '/ebc_data/nf-core/conda' - } process { executor = 'slurm' conda = "$baseDir/environment.yml" beforeScript = 'module load nextflow' + queue = { task.time <= 24.h ? 'testing' : 'main' } } executor { queueSize = 16 From 8e3803e4d7dc8b95f15a084f4e9790bd8731fe5b Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 16:57:45 +0200 Subject: [PATCH 21/22] Update ebc.config --- conf/ebc.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/ebc.config b/conf/ebc.config index ea48f05..4df0596 100644 --- a/conf/ebc.config +++ b/conf/ebc.config @@ -14,7 +14,6 @@ executor = 'slurm' conda = "$baseDir/environment.yml" beforeScript = 'module load nextflow' - queue = { task.time <= 24.h ? 'testing' : 'main' } } executor { queueSize = 16 From dcb478c904d35b3b49db20abc4a2ecdf990e3dc1 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Mon, 9 Mar 2020 17:07:03 +0200 Subject: [PATCH 22/22] Update ebc.md --- docs/ebc.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ebc.md b/docs/ebc.md index 45c9415..71f2622 100644 --- a/docs/ebc.md +++ b/docs/ebc.md @@ -3,6 +3,8 @@ All nf-core pipelines have been successfully configured for use on the [Estonian Biocentre (EBC)](https://genomics.ut.ee/en/about-us/estonian-biocentre) cluster at the [High Performance Computing Center](https://hpc.ut.ee/en) of the the University of Tartu. To use, run the pipeline with `-profile ebc`. This will download and launch the [`ebc.config`](../conf/ebc.config) which has been pre-configured with a setup suitable for the EBC cluster. Using this profile, currently, a conda environment containing all of the required software will be downloaded and stored in a central location. +> :warning: You must install your own [conda binary](conda.io) to run nf-core pipelines in a conda environment. Running with singularity will be added soon. + The profile will put a maximum job limit of 12 GB, 20 CPUs and a maximum wall time of 120 hours. NB: You will need an account to use the HPC cluster on EBC cluster in order to run the pipeline. If in doubt contact IT.