From b2f0a1f65342e986e759216f424e3e7162944a33 Mon Sep 17 00:00:00 2001 From: Keiran Raine Date: Fri, 17 Feb 2017 16:16:02 +0000 Subject: [PATCH 1/8] More streamlining of build --- build/apt-build.sh | 2 +- build/opt-build.sh | 225 +++++++++++++++++++++++++++++++++-------- build/perllib-build.sh | 15 +-- 3 files changed, 186 insertions(+), 56 deletions(-) diff --git a/build/apt-build.sh b/build/apt-build.sh index 0ef63a4..46cb5df 100644 --- a/build/apt-build.sh +++ b/build/apt-build.sh @@ -2,4 +2,4 @@ set -eux -echo "I currently do nothing, inplace to save build flow changes if needed later" +apt-get install -qy --no-install-recommends python diff --git a/build/opt-build.sh b/build/opt-build.sh index a7bf949..03e467e 100644 --- a/build/opt-build.sh +++ b/build/opt-build.sh @@ -8,53 +8,196 @@ fi set -u -mkdir -p $TMPDIR/downloads - -cd $TMPDIR/downloads - # cgpVcf -curl -sSL -o distro.zip --retry 10 https://github.com/cancerit/cgpVcf/archive/v2.1.1.zip -mkdir $TMPDIR/downloads/distro -bsdtar -C $TMPDIR/downloads/distro --strip-components 1 -xf distro.zip -cd $TMPDIR/downloads/distro -./setup.sh $OPT -cd $TMPDIR/downloads -rm -rf distro.zip $TMPDIR/downloads/distro /tmp/hts_cache +VER_CGPVCF="v2.2.0" +VER_VCFTOOLS="0.1.14" # cgpPindel -curl -sSL -o distro.zip --retry 10 https://github.com/cancerit/cgpPindel/archive/v2.1.0.zip -mkdir $TMPDIR/downloads/distro -bsdtar -C $TMPDIR/downloads/distro --strip-components 1 -xf distro.zip -cd $TMPDIR/downloads/distro -./setup.sh $OPT -cd $TMPDIR/downloads -rm -rf distro.zip $TMPDIR/downloads/distro /tmp/hts_cache +VER_CGPPINDEL="v2.1.0" # cgpCaVEManPostProcessing -curl -sSL -o distro.zip --retry 10 https://github.com/cancerit/cgpCaVEManPostProcessing/archive/1.6.6.zip -mkdir $TMPDIR/downloads/distro -bsdtar -C $TMPDIR/downloads/distro --strip-components 1 -xf distro.zip -cd $TMPDIR/downloads/distro -./setup.sh $OPT -cd $TMPDIR/downloads -rm -rf distro.zip $TMPDIR/downloads/distro /tmp/hts_cache +VER_CGPCAVEPOSTPROC="1.6.8" +VER_BEDTOOLS="2.21.0" # cgpCaVEManWrapper -curl -sSL -o distro.zip --retry 10 https://github.com/cancerit/cgpCaVEManWrapper/archive/1.10.3.zip -mkdir $TMPDIR/downloads/distro -bsdtar -C $TMPDIR/downloads/distro --strip-components 1 -xf distro.zip -cd $TMPDIR/downloads/distro -./setup.sh $OPT -cd $TMPDIR/downloads -rm -rf distro.zip $TMPDIR/downloads/distro /tmp/hts_cache +VER_CGPCAVEWRAP="1.10.3" +VER_CAVEMAN="1.11.3" # VAGrENT -curl -sSL -o distro.zip --retry 10 https://github.com/cancerit/VAGrENT/archive/v3.2.1.zip -mkdir $TMPDIR/downloads/distro -bsdtar -C $TMPDIR/downloads/distro --strip-components 1 -xf distro.zip -cd $TMPDIR/downloads/distro -./setup.sh $OPT -cd $TMPDIR/downloads -rm -rf distro.zip $TMPDIR/downloads/distro /tmp/hts_cache - -rm -rf $TMPDIR/downloads +VER_VAGRENT="v3.2.1" + +if [ "$#" -lt "1" ] ; then + echo "Please provide an installation path such as /opt/ICGC" + exit 1 +fi + + +# get path to this script +SCRIPT_PATH=`dirname $0`; +SCRIPT_PATH=`(cd $SCRIPT_PATH && pwd)` + +# get the location to install to +INST_PATH=$1 +mkdir -p $1 +INST_PATH=`(cd $1 && pwd)` +echo $INST_PATH + +# get current directory +INIT_DIR=`pwd` + +CPU=`grep -c ^processor /proc/cpuinfo` +if [ $? -eq 0 ]; then + if [ "$CPU" -gt "6" ]; then + CPU=6 + fi +else + CPU=1 +fi +echo "Max compilation CPUs set to $CPU" + + +SETUP_DIR=$INIT_DIR/install_tmp +mkdir -p $SETUP_DIR/distro # don't delete the actual distro directory until the very end +mkdir -p $INST_PATH/bin +cd $SETUP_DIR + +# make sure tools installed can see the install loc of libraries +set +u +export LD_LIBRARY_PATH=`echo $INST_PATH/lib:$LD_LIBRARY_PATH | perl -pe 's/:\$//;'` +export PATH=`echo $INST_PATH/bin:$PATH | perl -pe 's/:\$//;'` +export MANPATH=`echo $INST_PATH/man:$INST_PATH/share/man:$MANPATH | perl -pe 's/:\$//;'` +export PERL5LIB=`echo $INST_PATH/lib/perl5:$PERL5LIB | perl -pe 's/:\$//;'` +set -u + +## vcftools +if [ ! -e $SETUP_DIR/vcftools.success ]; then + curl -sSL --retry 10 https://github.com/vcftools/vcftools/releases/download/v${VER_VCFTOOLS}/vcftools-${VER_VCFTOOLS}.tar.gz > distro.tar.gz + rm -rf distro/* + tar --strip-components 1 -C distro -xzf distro.tar.gz + cd distro + curl -sSL https://raw.githubusercontent.com/cancerit/cgpVcf/${VER_CGPVCF}/patches/vcfToolsProcessLog.diff | patch src/perl/Vcf.pm + ./configure --prefix=$INST_PATH --with-pmdir=$INST_PATH/lib/perl5 + make -j$CPU + make install + cd $SETUP_DIR + rm -rf distro.* distro/* + touch $SETUP_DIR/vcftools.success +fi + +### cgpVcf +if [ ! -e $SETUP_DIR/cgpVcf.success ]; then + curl -sSL --retry 10 https://github.com/cancerit/cgpVcf/archive/${VER_CGPVCF}.tar.gz > distro.tar.gz + rm -rf distro/* + tar --strip-components 1 -C distro -xzf distro.tar.gz + cd distro + cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps . + cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH . + cd $SETUP_DIR + rm -rf distro.* distro/* + touch $SETUP_DIR/cgpVcf.success +fi + +### cgpPindel +if [ ! -e $SETUP_DIR/cgpPindel.success ]; then + curl -sSL --retry 10 https://github.com/cancerit/cgpPindel/archive/${VER_CGPPINDEL}.tar.gz > distro.tar.gz + rm -rf distro/* + tar --strip-components 1 -C distro -xzf distro.tar.gz + cd distro + if [ ! -e $SETUP_DIR/cgpPindel_c.success ]; then + g++ -O3 -o $INST_PATH/bin/pindel c++/pindel.cpp + g++ -O3 -o $INST_PATH/bin/filter_pindel_reads c++/filter_pindel_reads.cpp + touch $SETUP_DIR/cgpPindel_c.success + fi + cd perl + cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps . + cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH . + cd $SETUP_DIR + rm -rf distro.* distro/* + touch $SETUP_DIR/cgpPindel.success +fi + +### bedtools for cgpCaVEManPostProcessing +if [ ! -e $SETUP_DIR/bedtools.success ]; then + curl -sSL --retry 10 https://github.com/arq5x/bedtools2/releases/download/v${VER_BEDTOOLS}/bedtools-${VER_BEDTOOLS}.tar.gz > distro.tar.gz + rm -rf distro/* + tar --strip-components 1 -C distro -xzf distro.tar.gz + make -C distro -j$CPU + cp distro/bin/* $INST_PATH/bin/. + cd $SETUP_DIR + rm -rf distro.* distro/* + touch $SETUP_DIR/bedtools.success +fi + +### cgpCaVEManPostProcessing +if [ ! -e $SETUP_DIR/cgpCaVEManPostProcessing.success ]; then + cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH File::ShareDir::Install + curl -sSL --retry 10 https://github.com/cancerit/cgpCaVEManPostProcessing/archive/${VER_CGPCAVEPOSTPROC}.tar.gz > distro.tar.gz + rm -rf distro/* + tar --strip-components 1 -C distro -xzf distro.tar.gz + cd distro + cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps . + cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH . + cd $SETUP_DIR + rm -rf distro.* distro/* + touch $SETUP_DIR/cgpCaVEManPostProcessing.success +fi + +### CaVEMan for cgpCaVEManWrapper +if [ ! -e $SETUP_DIR/CaVEMan.success ]; then + curl -sSL --retry 10 https://github.com/cancerit/CaVEMan/archive/${VER_CAVEMAN}.tar.gz > distro.tar.gz + rm -rf distro/* + tar --strip-components 1 -C distro -xzf distro.tar.gz + cd distro + mkdir -p c/bin + + make clean + make -j$CPU prefix=$INST_PATH + cp bin/caveman $INST_PATH/bin/. + cp bin/mergeCavemanResults $INST_PATH/bin/. + cd $SETUP_DIR + rm -rf distro.* distro/* + touch $SETUP_DIR/CaVEMan.success +fi + +### cgpCaVEManWrapper +if [ ! -e $SETUP_DIR/cgpCaVEManWrapper.success ]; then + curl -sSL --retry 10 https://github.com/cancerit/cgpCaVEManWrapper/archive/${VER_CGPCAVEWRAP}.tar.gz > distro.tar.gz + rm -rf distro/* + tar --strip-components 1 -C distro -xzf distro.tar.gz + cd distro + cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps . + cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH . + cd $SETUP_DIR + rm -rf distro.* distro/* + touch $SETUP_DIR/cgpCaVEManWrapper.success +fi + +### VAGrENT +if [ ! -e $SETUP_DIR/VAGrENT.success ]; then + curl -sSL --retry 10 https://github.com/cancerit/VAGrENT/archive/${VER_VAGRENT}.tar.gz > distro.tar.gz + rm -rf distro/* + tar --strip-components 1 -C distro -xzf distro.tar.gz + cd distro + cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps . + cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH . + cd $SETUP_DIR + rm -rf distro.* distro/* + touch $SETUP_DIR/VAGrENT.success +fi + +cd $HOME +rm -rf $SETUP_DIR + +set +x + +echo " +################################################################ + + To use the non-central tools you need to set the following + export LD_LIBRARY_PATH=$INST_PATH/lib:\$LD_LIBRARY_PATH + export PATH=$INST_PATH/bin:\$PATH + export MANPATH=$INST_PATH/man:$INST_PATH/share/man:\$MANPATH + export PERL5LIB=$INST_PATH/lib/perl5:\$PERL5LIB + +################################################################ +" diff --git a/build/perllib-build.sh b/build/perllib-build.sh index 47f21ef..782620a 100644 --- a/build/perllib-build.sh +++ b/build/perllib-build.sh @@ -2,17 +2,4 @@ set -eux -apt-get -yq update -# install all perl libs, identify by grep of build "grep 'Successfully installed' build.log" -# much faster, items needing later versions will still upgrade -# still install those that get an upgrade though as dependancies will be resolved - -apt-get -yq install libdist-checkconflicts-perl -apt-get -yq install libeval-closure-perl -apt-get -yq install libclass-singleton-perl -apt-get -yq install libscalar-list-utils-perl -apt-get -yq install libdatetime-locale-perl -apt-get -yq install libdatetime-perl -apt-get -yq install liblog-log4perl-perl -apt-get -yq install libfile-type-perl -apt-get -yq install libsort-key-perl +echo "Doing nothing, here for perl libs that won't install through cpanminus" From 1565485afc0ff44a9d4b5fb28e13abbed574130f Mon Sep 17 00:00:00 2001 From: Keiran Raine Date: Fri, 17 Feb 2017 16:19:17 +0000 Subject: [PATCH 2/8] Missing params --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b1b7adc..f1fa9d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,7 @@ USER root ENV OPT /opt/wtsi-cgp ENV PATH $OPT/bin:$PATH ENV PERL5LIB $OPT/lib/perl5 +ENV LD_LIBRARY_PATH $OPT/lib ADD build/apt-build.sh build/ RUN bash build/apt-build.sh @@ -19,7 +20,7 @@ ADD build/perllib-build.sh build/ RUN bash build/perllib-build.sh ADD build/opt-build.sh build/ -RUN bash build/opt-build.sh +RUN bash build/opt-build.sh $OPT ADD scripts/analysisWXS.sh $OPT/bin/analysisWXS.sh ADD scripts/ds-wrapper.pl $OPT/bin/ds-wrapper.pl From 5a6aa3748fcbe8e950ca26f3b73e6842f9616fd0 Mon Sep 17 00:00:00 2001 From: Keiran Raine Date: Tue, 21 Feb 2017 20:57:30 +0000 Subject: [PATCH 3/8] Update base image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f1fa9d7..c47fb56 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/wtsicgp/dockstore-cgpmap:1.0.8 +FROM quay.io/wtsicgp/dockstore-cgpmap:2.0.0 MAINTAINER keiranmraine@gmail.com From ba7dd28bcfab83e396863f7808020d95d0473dd9 Mon Sep 17 00:00:00 2001 From: Keiran Raine Date: Wed, 22 Feb 2017 14:51:32 +0000 Subject: [PATCH 4/8] missing setting of params --- scripts/analysisWXS.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/analysisWXS.sh b/scripts/analysisWXS.sh index a1905f1..292eade 100755 --- a/scripts/analysisWXS.sh +++ b/scripts/analysisWXS.sh @@ -182,7 +182,9 @@ do_parallel[CaVEMan]="caveman.pl \ -nb $BAM_WT_TMP \ -c $SNVFLAG \ -f $REF_BASE/caveman/flagging/flag.to.vcf.convert.ini \ - -o $OUTPUT_DIR/${NAME_MT}_vs_${NAME_WT}/caveman" + -o $OUTPUT_DIR/${NAME_MT}_vs_${NAME_WT}/caveman \ + -np $PROTOCOL \ + -wp $PROTOCOL" echo "Starting Parallel block 2: `date`" run_parallel $CPU do_parallel From 977df10f275eb040290885f691fb3647e5600848 Mon Sep 17 00:00:00 2001 From: Keiran Raine Date: Wed, 22 Feb 2017 14:51:59 +0000 Subject: [PATCH 5/8] Pull in new caveman wrapper to handle protocol/seqType mismatch --- build/opt-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/opt-build.sh b/build/opt-build.sh index 03e467e..ef7046a 100644 --- a/build/opt-build.sh +++ b/build/opt-build.sh @@ -20,7 +20,7 @@ VER_CGPCAVEPOSTPROC="1.6.8" VER_BEDTOOLS="2.21.0" # cgpCaVEManWrapper -VER_CGPCAVEWRAP="1.10.3" +VER_CGPCAVEWRAP="1.10.4" VER_CAVEMAN="1.11.3" # VAGrENT From 26902c93f49a87e6a652847e0896c475c122e11e Mon Sep 17 00:00:00 2001 From: Keiran Raine Date: Wed, 22 Feb 2017 14:58:55 +0000 Subject: [PATCH 6/8] typo on param name --- scripts/analysisWXS.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/analysisWXS.sh b/scripts/analysisWXS.sh index 292eade..1d82bbb 100755 --- a/scripts/analysisWXS.sh +++ b/scripts/analysisWXS.sh @@ -184,7 +184,7 @@ do_parallel[CaVEMan]="caveman.pl \ -f $REF_BASE/caveman/flagging/flag.to.vcf.convert.ini \ -o $OUTPUT_DIR/${NAME_MT}_vs_${NAME_WT}/caveman \ -np $PROTOCOL \ - -wp $PROTOCOL" + -tp $PROTOCOL" echo "Starting Parallel block 2: `date`" run_parallel $CPU do_parallel From ca41b8f29c6d3fa462ba15e90f8a07c1550fb80f Mon Sep 17 00:00:00 2001 From: Keiran Raine Date: Wed, 22 Feb 2017 16:02:44 +0000 Subject: [PATCH 7/8] Modifies fully configured flag config so that is always a useable file in ref set when run outside of framework --- scripts/ds-wrapper.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ds-wrapper.pl b/scripts/ds-wrapper.pl index 0fc0b1f..a3af580 100644 --- a/scripts/ds-wrapper.pl +++ b/scripts/ds-wrapper.pl @@ -102,11 +102,12 @@ sub add_species_flag_ini { my ($species, $ini_in) = @_; $species =~ s/ /_/g; + $species = uc $species; my $ini_out = $ENV{HOME}.'/flag.vcf.config.WXS.ini'; open my $IN, '<', $ini_in; open my $OUT,'>',$ini_out; while(my $line = <$IN>) { - $line =~ s/^\[/[$species/; + $line =~ s/^\[HUMAN_/[${species}_/; print $OUT $line; } close $OUT; From 192188d68f04d3cce6fdf84feacc60716b67d390 Mon Sep 17 00:00:00 2001 From: Keiran Raine Date: Wed, 22 Feb 2017 16:21:26 +0000 Subject: [PATCH 8/8] Release notes and version bump --- CHANGES.md | 21 +++++++++++++++++++++ Dockerfile | 2 +- Dockstore.cwl | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 CHANGES.md diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..10eab9c --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,21 @@ +### 2.0.0 +* See [dockstore-cgp:2.0.0](https://github.com/cancerit/dockstore-cgpmap/releases/tag/2.0.0) +* Streamlined install process to reduce build time and size of image. +* Fix to SNV flagging - was not being applied previously. + +### 1.0.4 +* Updates base image to add load management facilities via ENV variables. + +### 1.0.3 +* Add params file to output bindings + +### 1.0.2 +* Fix up output files slightly +* Update base image for dependency (PCAP-core etc) + +### 1.0.1 +* Fixes handling of species/assembly +* Update base image (PCAP-core etc) + +### 1.0.0 +Initial release of flow diff --git a/Dockerfile b/Dockerfile index c47fb56..d0db456 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM quay.io/wtsicgp/dockstore-cgpmap:2.0.0 MAINTAINER keiranmraine@gmail.com LABEL uk.ac.sanger.cgp="Cancer Genome Project, Wellcome Trust Sanger Institute" \ - version="1.0.4" \ + version="2.0.0" \ description="The CGP WXS pipeline for dockstore.org" USER root diff --git a/Dockstore.cwl b/Dockstore.cwl index 4392803..767852a 100644 --- a/Dockstore.cwl +++ b/Dockstore.cwl @@ -19,7 +19,7 @@ dct:creator: requirements: - class: DockerRequirement - dockerPull: "quay.io/wtsicgp/dockstore-cgpwxs:1.0.4" + dockerPull: "quay.io/wtsicgp/dockstore-cgpwxs:2.0.0" hints: - class: ResourceRequirement