Merge branch 'release/2.0.0'

CHANGES.md

@@ -0,0 +1,21 @@
+### 2.0.0
+* See [dockstore-cgp:2.0.0](https://github.com/cancerit/dockstore-cgpmap/releases/tag/2.0.0)
+* Streamlined install process to reduce build time and size of image.
+* Fix to SNV flagging - was not being applied previously.
+
+### 1.0.4
+* Updates base image to add load management facilities via ENV variables.
+
+### 1.0.3
+* Add params file to output bindings
+
+### 1.0.2
+* Fix up output files slightly
+* Update base image for dependency (PCAP-core etc)
+
+### 1.0.1
+* Fixes handling of species/assembly
+* Update base image (PCAP-core etc)
+
+### 1.0.0
+Initial release of flow

Dockerfile

@@ -1,16 +1,17 @@
-FROM  quay.io/wtsicgp/dockstore-cgpmap:1.0.8
+FROM  quay.io/wtsicgp/dockstore-cgpmap:2.0.0
 
 MAINTAINER  keiranmraine@gmail.com
 
 LABEL uk.ac.sanger.cgp="Cancer Genome Project, Wellcome Trust Sanger Institute" \
-      version="1.0.4" \
+      version="2.0.0" \
       description="The CGP WXS pipeline for dockstore.org"
 
 USER  root
 
 ENV OPT /opt/wtsi-cgp
 ENV PATH $OPT/bin:$PATH
 ENV PERL5LIB $OPT/lib/perl5
+ENV LD_LIBRARY_PATH $OPT/lib
 
 ADD build/apt-build.sh build/
 RUN bash build/apt-build.sh
@@ -19,7 +20,7 @@ ADD build/perllib-build.sh build/
 RUN bash build/perllib-build.sh
 
 ADD build/opt-build.sh build/
-RUN bash build/opt-build.sh
+RUN bash build/opt-build.sh $OPT
 
 ADD scripts/analysisWXS.sh $OPT/bin/analysisWXS.sh
 ADD scripts/ds-wrapper.pl $OPT/bin/ds-wrapper.pl

Dockstore.cwl

@@ -19,7 +19,7 @@ dct:creator:
 
 requirements:
   - class: DockerRequirement
-    dockerPull: "quay.io/wtsicgp/dockstore-cgpwxs:1.0.4"
+    dockerPull: "quay.io/wtsicgp/dockstore-cgpwxs:2.0.0"
 
 hints:
   - class: ResourceRequirement

build/apt-build.sh

@@ -2,4 +2,4 @@
 
 set -eux
 
-echo "I currently do nothing, inplace to save build flow changes if needed later"
+apt-get install -qy --no-install-recommends python

build/opt-build.sh

@@ -8,53 +8,196 @@ fi
 
 set -u
 
-mkdir -p $TMPDIR/downloads
-
-cd $TMPDIR/downloads
-
 # cgpVcf
-curl -sSL -o distro.zip --retry 10 https://github.com/cancerit/cgpVcf/archive/v2.1.1.zip
-mkdir $TMPDIR/downloads/distro
-bsdtar -C $TMPDIR/downloads/distro --strip-components 1 -xf distro.zip
-cd $TMPDIR/downloads/distro
-./setup.sh $OPT
-cd $TMPDIR/downloads
-rm -rf distro.zip $TMPDIR/downloads/distro /tmp/hts_cache
+VER_CGPVCF="v2.2.0"
+VER_VCFTOOLS="0.1.14"
 
 # cgpPindel
-curl -sSL -o distro.zip --retry 10 https://github.com/cancerit/cgpPindel/archive/v2.1.0.zip
-mkdir $TMPDIR/downloads/distro
-bsdtar -C $TMPDIR/downloads/distro --strip-components 1 -xf distro.zip
-cd $TMPDIR/downloads/distro
-./setup.sh $OPT
-cd $TMPDIR/downloads
-rm -rf distro.zip $TMPDIR/downloads/distro /tmp/hts_cache
+VER_CGPPINDEL="v2.1.0"
 
 # cgpCaVEManPostProcessing
-curl -sSL -o distro.zip --retry 10 https://github.com/cancerit/cgpCaVEManPostProcessing/archive/1.6.6.zip
-mkdir $TMPDIR/downloads/distro
-bsdtar -C $TMPDIR/downloads/distro --strip-components 1 -xf distro.zip
-cd $TMPDIR/downloads/distro
-./setup.sh $OPT
-cd $TMPDIR/downloads
-rm -rf distro.zip $TMPDIR/downloads/distro /tmp/hts_cache
+VER_CGPCAVEPOSTPROC="1.6.8"
+VER_BEDTOOLS="2.21.0"
 
 # cgpCaVEManWrapper
-curl -sSL -o distro.zip --retry 10 https://github.com/cancerit/cgpCaVEManWrapper/archive/1.10.3.zip
-mkdir $TMPDIR/downloads/distro
-bsdtar -C $TMPDIR/downloads/distro --strip-components 1 -xf distro.zip
-cd $TMPDIR/downloads/distro
-./setup.sh $OPT
-cd $TMPDIR/downloads
-rm -rf distro.zip $TMPDIR/downloads/distro /tmp/hts_cache
+VER_CGPCAVEWRAP="1.10.4"
+VER_CAVEMAN="1.11.3"
 
 # VAGrENT
-curl -sSL -o distro.zip --retry 10 https://github.com/cancerit/VAGrENT/archive/v3.2.1.zip
-mkdir $TMPDIR/downloads/distro
-bsdtar -C $TMPDIR/downloads/distro --strip-components 1 -xf distro.zip
-cd $TMPDIR/downloads/distro
-./setup.sh $OPT
-cd $TMPDIR/downloads
-rm -rf distro.zip $TMPDIR/downloads/distro /tmp/hts_cache
-
-rm -rf $TMPDIR/downloads
+VER_VAGRENT="v3.2.1"
+
+if [ "$#" -lt "1" ] ; then
+  echo "Please provide an installation path such as /opt/ICGC"
+  exit 1
+fi
+
+
+# get path to this script
+SCRIPT_PATH=`dirname $0`;
+SCRIPT_PATH=`(cd $SCRIPT_PATH && pwd)`
+
+# get the location to install to
+INST_PATH=$1
+mkdir -p $1
+INST_PATH=`(cd $1 && pwd)`
+echo $INST_PATH
+
+# get current directory
+INIT_DIR=`pwd`
+
+CPU=`grep -c ^processor /proc/cpuinfo`
+if [ $? -eq 0 ]; then
+  if [ "$CPU" -gt "6" ]; then
+    CPU=6
+  fi
+else
+  CPU=1
+fi
+echo "Max compilation CPUs set to $CPU"
+
+
+SETUP_DIR=$INIT_DIR/install_tmp
+mkdir -p $SETUP_DIR/distro # don't delete the actual distro directory until the very end
+mkdir -p $INST_PATH/bin
+cd $SETUP_DIR
+
+# make sure tools installed can see the install loc of libraries
+set +u
+export LD_LIBRARY_PATH=`echo $INST_PATH/lib:$LD_LIBRARY_PATH | perl -pe 's/:\$//;'`
+export PATH=`echo $INST_PATH/bin:$PATH | perl -pe 's/:\$//;'`
+export MANPATH=`echo $INST_PATH/man:$INST_PATH/share/man:$MANPATH | perl -pe 's/:\$//;'`
+export PERL5LIB=`echo $INST_PATH/lib/perl5:$PERL5LIB | perl -pe 's/:\$//;'`
+set -u
+
+## vcftools
+if [ ! -e $SETUP_DIR/vcftools.success ]; then
+  curl -sSL --retry 10 https://github.com/vcftools/vcftools/releases/download/v${VER_VCFTOOLS}/vcftools-${VER_VCFTOOLS}.tar.gz > distro.tar.gz
+  rm -rf distro/*
+  tar --strip-components 1 -C distro -xzf distro.tar.gz
+  cd distro
+  curl -sSL https://raw.githubusercontent.com/cancerit/cgpVcf/${VER_CGPVCF}/patches/vcfToolsProcessLog.diff | patch src/perl/Vcf.pm
+  ./configure --prefix=$INST_PATH --with-pmdir=$INST_PATH/lib/perl5
+  make -j$CPU
+  make install
+  cd $SETUP_DIR
+  rm -rf distro.* distro/*
+  touch $SETUP_DIR/vcftools.success
+fi
+
+### cgpVcf
+if [ ! -e $SETUP_DIR/cgpVcf.success ]; then
+  curl -sSL --retry 10 https://github.com/cancerit/cgpVcf/archive/${VER_CGPVCF}.tar.gz > distro.tar.gz
+  rm -rf distro/*
+  tar --strip-components 1 -C distro -xzf distro.tar.gz
+  cd distro
+  cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps .
+  cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH .
+  cd $SETUP_DIR
+  rm -rf distro.* distro/*
+  touch $SETUP_DIR/cgpVcf.success
+fi
+
+### cgpPindel
+if [ ! -e $SETUP_DIR/cgpPindel.success ]; then
+  curl -sSL --retry 10 https://github.com/cancerit/cgpPindel/archive/${VER_CGPPINDEL}.tar.gz > distro.tar.gz
+  rm -rf distro/*
+  tar --strip-components 1 -C distro -xzf distro.tar.gz
+  cd distro
+  if [ ! -e $SETUP_DIR/cgpPindel_c.success ]; then
+    g++ -O3 -o $INST_PATH/bin/pindel c++/pindel.cpp
+    g++ -O3 -o $INST_PATH/bin/filter_pindel_reads c++/filter_pindel_reads.cpp
+    touch $SETUP_DIR/cgpPindel_c.success
+  fi
+  cd perl
+  cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps .
+  cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH .
+  cd $SETUP_DIR
+  rm -rf distro.* distro/*
+  touch $SETUP_DIR/cgpPindel.success
+fi
+
+### bedtools for cgpCaVEManPostProcessing
+if [ ! -e $SETUP_DIR/bedtools.success ]; then
+  curl -sSL --retry 10 https://github.com/arq5x/bedtools2/releases/download/v${VER_BEDTOOLS}/bedtools-${VER_BEDTOOLS}.tar.gz > distro.tar.gz
+  rm -rf distro/*
+  tar --strip-components 1 -C distro -xzf distro.tar.gz
+  make -C distro -j$CPU
+  cp distro/bin/* $INST_PATH/bin/.
+  cd $SETUP_DIR
+  rm -rf distro.* distro/*
+  touch $SETUP_DIR/bedtools.success
+fi
+
+### cgpCaVEManPostProcessing
+if [ ! -e $SETUP_DIR/cgpCaVEManPostProcessing.success ]; then
+  cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH File::ShareDir::Install
+  curl -sSL --retry 10 https://github.com/cancerit/cgpCaVEManPostProcessing/archive/${VER_CGPCAVEPOSTPROC}.tar.gz > distro.tar.gz
+  rm -rf distro/*
+  tar --strip-components 1 -C distro -xzf distro.tar.gz
+  cd distro
+  cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps .
+  cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH .
+  cd $SETUP_DIR
+  rm -rf distro.* distro/*
+  touch $SETUP_DIR/cgpCaVEManPostProcessing.success
+fi
+
+### CaVEMan for cgpCaVEManWrapper
+if [ ! -e $SETUP_DIR/CaVEMan.success ]; then
+  curl -sSL --retry 10 https://github.com/cancerit/CaVEMan/archive/${VER_CAVEMAN}.tar.gz > distro.tar.gz
+  rm -rf distro/*
+  tar --strip-components 1 -C distro -xzf distro.tar.gz
+  cd distro
+  mkdir -p c/bin
+
+  make clean
+  make -j$CPU prefix=$INST_PATH
+  cp bin/caveman $INST_PATH/bin/.
+  cp bin/mergeCavemanResults $INST_PATH/bin/.
+  cd $SETUP_DIR
+  rm -rf distro.* distro/*
+  touch $SETUP_DIR/CaVEMan.success
+fi
+
+### cgpCaVEManWrapper
+if [ ! -e $SETUP_DIR/cgpCaVEManWrapper.success ]; then
+  curl -sSL --retry 10 https://github.com/cancerit/cgpCaVEManWrapper/archive/${VER_CGPCAVEWRAP}.tar.gz > distro.tar.gz
+  rm -rf distro/*
+  tar --strip-components 1 -C distro -xzf distro.tar.gz
+  cd distro
+  cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps .
+  cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH .
+  cd $SETUP_DIR
+  rm -rf distro.* distro/*
+  touch $SETUP_DIR/cgpCaVEManWrapper.success
+fi
+
+### VAGrENT
+if [ ! -e $SETUP_DIR/VAGrENT.success ]; then
+  curl -sSL --retry 10 https://github.com/cancerit/VAGrENT/archive/${VER_VAGRENT}.tar.gz > distro.tar.gz
+  rm -rf distro/*
+  tar --strip-components 1 -C distro -xzf distro.tar.gz
+  cd distro
+  cpanm --no-interactive --notest --mirror http://cpan.metacpan.org --notest -l $INST_PATH --installdeps .
+  cpanm -v --no-interactive --mirror http://cpan.metacpan.org -l $INST_PATH .
+  cd $SETUP_DIR
+  rm -rf distro.* distro/*
+  touch $SETUP_DIR/VAGrENT.success
+fi
+
+cd $HOME
+rm -rf $SETUP_DIR
+
+set +x
+
+echo "
+################################################################
+
+  To use the non-central tools you need to set the following
+    export LD_LIBRARY_PATH=$INST_PATH/lib:\$LD_LIBRARY_PATH
+    export PATH=$INST_PATH/bin:\$PATH
+    export MANPATH=$INST_PATH/man:$INST_PATH/share/man:\$MANPATH
+    export PERL5LIB=$INST_PATH/lib/perl5:\$PERL5LIB
+
+################################################################
+"

build/perllib-build.sh

@@ -2,17 +2,4 @@
 
 set -eux
 
-apt-get -yq update
-# install all perl libs, identify by grep of build "grep 'Successfully installed' build.log"
-# much faster, items needing later versions will still upgrade
-# still install those that get an upgrade though as dependancies will be resolved
-
-apt-get -yq install libdist-checkconflicts-perl
-apt-get -yq install libeval-closure-perl
-apt-get -yq install libclass-singleton-perl
-apt-get -yq install libscalar-list-utils-perl
-apt-get -yq install libdatetime-locale-perl
-apt-get -yq install libdatetime-perl
-apt-get -yq install liblog-log4perl-perl
-apt-get -yq install libfile-type-perl
-apt-get -yq install libsort-key-perl
+echo "Doing nothing, here for perl libs that won't install through cpanminus"

scripts/analysisWXS.sh

@@ -182,7 +182,9 @@ do_parallel[CaVEMan]="caveman.pl \
  -nb $BAM_WT_TMP \
  -c $SNVFLAG \
  -f $REF_BASE/caveman/flagging/flag.to.vcf.convert.ini \
- -o $OUTPUT_DIR/${NAME_MT}_vs_${NAME_WT}/caveman"
+ -o $OUTPUT_DIR/${NAME_MT}_vs_${NAME_WT}/caveman \
+ -np $PROTOCOL \
+ -tp $PROTOCOL"
 
 echo "Starting Parallel block 2: `date`"
 run_parallel $CPU do_parallel

scripts/ds-wrapper.pl

@@ -102,11 +102,12 @@
 sub add_species_flag_ini {
   my ($species, $ini_in) = @_;
   $species =~ s/ /_/g;
+  $species = uc $species;
   my $ini_out = $ENV{HOME}.'/flag.vcf.config.WXS.ini';
   open my $IN, '<', $ini_in;
   open my $OUT,'>',$ini_out;
   while(my $line = <$IN>) {
-    $line =~ s/^\[/[$species/;
+    $line =~ s/^\[HUMAN_/[${species}_/;
     print $OUT $line;
   }
   close $OUT;