Merge pull request #290 from wtsi-npg/devel

prep for release 0.30.0

.github/workflows/testing_and_building_repo.yml

@@ -0,0 +1,69 @@
+name: testing_and_building_repo
+on: [push, pull_request]
+jobs:
+  build: 
+    strategy:
+      matrix:
+        os: ['ubuntu-18.04']
+
+    runs-on: ${{matrix.os}} 
+    name: Distribution Perl on ${{ matrix.os }} 
+    steps:
+    - uses: actions/checkout@v2 
+
+   # Caching cpanm external modules
+    - name: Cache cpanm external modules
+      id: cpanmCache
+      uses: actions/cache@v2
+      with:
+        path: ~/perl5ext  
+        key: ${{ matrix.os}}-build-cpanm-external
+
+    - name: install cpanm 
+      run: |
+          wget -qO - https://cpanmin.us | /usr/bin/perl - --sudo App::cpanminus
+
+    # run the perl install script
+    - name: install NPG Perl dependencies, and their CPAN dependencies
+      run: |
+          cpanm --local-lib=~/perl5ext local::lib && eval $(perl -I ~/perl5ext/lib/perl5/ -Mlocal::lib)
+          ${GITHUB_WORKSPACE}/scripts/install_npg_perl_dependencies.sh $WTSI_NPG_GITHUB_URL $WTSI_NPG_BUILD_BRANCH
+      env:
+        WTSI_NPG_GITHUB_URL: https://github.com/wtsi-npg 
+        WTSI_NPG_BUILD_BRANCH: ${GITHUB_HEAD_REF}
+
+    - name: install cpanm dependencies
+      run: |
+          eval $(perl -I ~/perl5ext/lib/perl5/ -Mlocal::lib=~/perl5npg)
+          eval $(perl -I ~/perl5ext/lib/perl5/ -Mlocal::lib=~/perl5ext)
+          cpanm --installdeps --notest .
+
+    - name: run Build.PL and ./Build
+      run: |
+          eval $(perl -I ~/perl5ext/lib/perl5/ -Mlocal::lib=~/perl5ext)
+          eval $(perl -I ~/perl5ext/lib/perl5/ -Mlocal::lib=~/perl5npg)
+          export TEST_AUTHOR=1
+          perl Build.PL
+          ./Build test --verbose
+          ./Build install
+
+    # running ./Build dist and exporting files
+    - name: run ./Build dist  
+      run: |
+          eval $(perl -I ~/perl5ext/lib/perl5/ -Mlocal::lib=~/perl5ext)
+          export TEST_AUTHOR=1          
+          ./Build dist
+          export DIST_FILE=$(ls p4-*.tar.gz)
+          export MD5_FILE=$DIST_FILE.md5
+          md5sum $DIST_FILE > $MD5_FILE
+          export SHA256_FILE=$DIST_FILE.sha256
+          shasum -a 256 $DIST_FILE > $SHA256_FILE
+   
+    # Archive logs if failure
+    - name: Archive CPAN logs
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: cpan_log
+        path: /home/runner/.cpanm/work/*/build.log
+        retention-days: 5

MANIFEST

@@ -1,3 +1,4 @@
+.github/workflows/testing_and_building_repo.yml
 bin/viv.pl
 bin/vtfp.pl
 Build.PL
@@ -7,36 +8,59 @@ data/bwa_post_proc.vtf
 data/hiseqx.vtf
 data/post_alignment.vtf
 data/vtlib/alignment_common.json
+data/vtlib/alignment_wtsi_stage2_humansplit_extrasplit_notargetalign_template.json
 data/vtlib/alignment_wtsi_stage2_humansplit_notargetalign_template.json
 data/vtlib/alignment_wtsi_stage2_humansplit_template.json
 data/vtlib/alignment_wtsi_stage2_template.json
 data/vtlib/auxmerge_prep.json
 data/vtlib/auxmerge_prep_realign.json
+data/vtlib/bamindexdecoder.json
 data/vtlib/basic_params_top_up_merge.json
+data/vtlib/bcftools_genotype_call.json
 data/vtlib/bcl2bam_phix_deplex_wtsi_stage1_template.json
 data/vtlib/bwa_aln_alignment.json
 data/vtlib/bwa_aln_se_alignment.json
 data/vtlib/bwa_mem_alignment.json
+data/vtlib/bwa_mem_bwakit_alignment.json
 data/vtlib/final_output_noalign_prep.json
 data/vtlib/final_output_prep.json
+data/vtlib/final_output_prep_chrsplit_noaln.json
+data/vtlib/final_stage1_process.json
+data/vtlib/final_stage1_process_nosplit.json
+data/vtlib/hisat2_alignment.json
+data/vtlib/markdup_biobambam.json
+data/vtlib/markdup_duplexseq.json
+data/vtlib/markdup_none.json
+data/vtlib/markdup_picard.json
+data/vtlib/markdup_samtools.json
 data/vtlib/merge_aligned.json
 data/vtlib/merge_final_output_prep.json
+data/vtlib/minimap2_alignment.json
+data/vtlib/passthrough.json
 data/vtlib/post_alignment.json
+data/vtlib/post_alignment_realignment.json
 data/vtlib/pre_alignment.json
 data/vtlib/pre_alignment_realign.json
 data/vtlib/README.vtlib
+data/vtlib/realignment_wtsi_humansplit_notargetalign_template.json
+data/vtlib/realignment_wtsi_humansplit_template.json
 data/vtlib/realignment_wtsi_template.json
 data/vtlib/salmon_alignment.json
 data/vtlib/seqchksum.json
+data/vtlib/seqchksum_ehs.json
 data/vtlib/seqchksum_hs.json
 data/vtlib/seqchksum_realign.json
+data/vtlib/seqchksum_stage1.json
 data/vtlib/split_by_chromosome.json
 data/vtlib/star_alignment.json
+data/vtlib/subsample.json
+data/vtlib/target_alignment.json
+data/vtlib/target_nchs_alignment.json
 data/vtlib/tophat2_alignment.json
-data/vtlib/markdup_biobambam.json
-data/vtlib/markdup_duplexseq.json
-data/vtlib/markdup_picard.json
-data/vtlib/markdup_samtools.json
+data/vtlib/unaln_intfile.json
+data/vtlib/unaln_intfile4.json
+data/vtlib/unaln_tee3.json
+data/vtlib/unaln_tee4.json
 examples/bwa_aln_cfg.png
 examples/bwa_mem/bwa_mem_alignment.vtf
 examples/bwa_mem/generic_alignment_with_phix.vtf
@@ -53,14 +77,40 @@ examples/tophat2_cfg.png
 MANIFEST			This list of files
 README
 README.vtfp
+scripts/install_npg_perl_dependencies.sh
 t/00-scripts_compile.t
+t/10-viv-noop_edge.t
+t/10-viv-packflag.t
+t/10-viv-stringify.t
+t/10-viv-teenode.t
 t/10-vtfp-array_expansion.t
+t/10-vtfp-noop.t
 t/10-vtfp-param_ring.t
 t/10-vtfp-pv.t
+t/10-vtfp-select_directive.t
+t/10-vtfp-select_directive.v2.t
+t/10-vtfp-splice_nodes.t
 t/10-vtfp-subst_directive.t
+t/10-vtfp-subst_directive.v2.t
+t/10-vtfp-vtfile.t
+t/10-vtfp-vtfile_v2.t
+t/50-viv.t
 t/data/10-vtfp-array_expansion.json
+t/data/10-vtfp-array_expansion.v2.json
+t/data/10-vtfp-noop.json
 t/data/10-vtfp-param_ring.json
 t/data/10-vtfp-pv.json
+t/data/10-vtfp-splice_nodes_00.json
+t/data/50-viv_failing_io_pipeline0.v2.vtf
+t/data/50-viv_failing_io_pipeline0.vtf
+t/data/50-viv_failing_io_pipeline1.v2.vtf
+t/data/50-viv_failing_io_pipeline1.vtf
+t/data/50-viv_failing_io_pipeline2.v2.vtf
+t/data/50-viv_failing_io_pipeline2.vtf
+t/data/50-viv_failing_pipeline.v2.vtf
+t/data/50-viv_failing_pipeline.vtf
+t/data/50-viv_pipeline.v2.vtf
+t/data/50-viv_pipeline.vtf
 visualisation/bwa_aln.cfg
 visualisation/bwa_aln.json
 visualisation/cgi-bin/getProgress

data/vtlib/alignment_common.json

@@ -15,6 +15,7 @@
 	{"id":"bwa_executable","required":"no","default":"bwa0_6"},
 	{"id":"aligner_numthreads","required":"no","default":"2"},
 	{"id":"bam_ext","required":"no","default":".bam"},
+	{"id":"bam_idx_ext","required":"no","default":".bai"},
 	{"id":"cram_ext","required":"no","default":".cram"},
 	{"id":"cram_idx_ext","required":"no","default":".cram.crai"},
         {"id":"alignment_method","required":"yes"},

data/vtlib/bwa_mem_alignment.json

@@ -33,7 +33,14 @@
 		"subst_constructor":{ "vals":[ "-K", {"subst":"bwa_mem_K_value"} ] }
 	},
 	{"id":"bwa_mem_p_flag","required":"no","default":"-p","comment":"by default, paired alignment is assumed"},
-	{"id":"bwa_mem_Y_flag","required":"no","default":"-Y","comment":"by default, supplementary alignment sequences will be soft clipped instead of hard clipped"}
+	{"id":"bwa_mem_Y_flag","required":"no","default":"-Y","comment":"by default, supplementary alignment sequences will be soft clipped instead of hard clipped"},
+	{"id":"bwa_mem_B_value","required":"no","comment":"if unspecified, -B flag is not used"},
+	{
+		"id":"bwa_mem_B_flag",
+		"comment":"penalty for a mismatch [default 4]",
+		"required":"no",
+		"subst_constructor":{ "vals":[ "-B", {"subst":"bwa_mem_B_value"} ] }
+	}
 ],
 "nodes":[
 	{
@@ -55,6 +62,11 @@
 			{"subst":"bwa_mem_p_flag"},{"subst":"bwa_mem_Y_flag"},
 			{"subst":"bwa_mem_T_flag"},
 			{"subst":"bwa_mem_K_flag"},
+			{"select":"bwa_mem_5_flag", "required":true, "select_range":[1], "default":"off", "cases":{"on":"-5","off":[]},"comment":"for split alignment, take the alignment with the smallest coordinate as primary"},
+			{"select":"bwa_mem_S_flag", "required":true, "select_range":[1], "default":"off", "cases":{"on":"-S","off":[]},"comment":"skip mate rescue"},
+			{"select":"bwa_mem_P_flag", "required":true, "select_range":[1], "default":"off", "cases":{"on":"-P","off":[]},"comment":"skip pairing; mate rescue performed unless -S also in use"},
+			{"subst":"bwa_mem_B_flag"},
+			{"subst":"bwa_mem_arbitrary_flags"},
 			{"port":"db_prefix_reference_genome", "direction":"in"},
 			{"port":"fq","direction":"in"}
 		]