Merge pull request #540 from ComparativeGenomicsToolkit/graphmap-split2

Prep next release (2.0.2)

README.md

@@ -40,10 +40,10 @@ There are many different ways to install and run Cactus:
 
 #### Docker Image
 
-Cactus docker images are hosted on [quay](https://quay.io/repository/comparative-genomics-toolkit/cactus).  The image for the latest release is listed on the [Releases Page](https://github.com/ComparativeGenomicsToolkit/cactus/releases).  Here is an command line to run the included evolver mammals example with release 2.0.1
+Cactus docker images are hosted on [quay](https://quay.io/repository/comparative-genomics-toolkit/cactus).  The image for the latest release is listed on the [Releases Page](https://github.com/ComparativeGenomicsToolkit/cactus/releases).  Here is an command line to run the included evolver mammals example with release 2.0.2
 ```
 wget https://raw.githubusercontent.com/ComparativeGenomicsToolkit/cactus/master/examples/evolverMammals.txt
-docker run -v $(pwd):/data --rm -it quay.io/comparative-genomics-toolkit/cactus:v2.0.1 cactus /data/jobStore /data/evolverMammals.txt /data/evolverMammals.hal --root mr --binariesMode local
+docker run -v $(pwd):/data --rm -it quay.io/comparative-genomics-toolkit/cactus:v2.0.2 cactus /data/jobStore /data/evolverMammals.txt /data/evolverMammals.hal --root mr --binariesMode local
 
 ```
 

ReleaseNotes.md

@@ -1,9 +1,16 @@
 # Release 2.0.2
 
+This release primarily addresses stability issues during pangenome construction.
+
 Changelog:
 - Use latest abpoa, which fixes bug where aligning >1024 sequences would lead to a segfault
 - Update to Toil 5.4.0
 - More consistently apply filters to minimap2 output in the fallback stage of graphmap-split
+- Build abpoa with AVX2 SIMD extensions instead of SSE4.1 in order to work around instability when building pangenomes.  This ups the hardware requirements for releases, unfortunately, as AVX2 is slightly newer.
+- Clean up CAF config parameters
+- Fix CAF secondary filter worst-case runtime issue.  It was very rare but could add days to runtime.
+- Slightly tune minimap2 thresholds used for chromosome splitting
+- Normalization option added to cactus-graphmap-join (should be used to work around soon-to-be addressed underalignment bug)
 
 # Release 2.0.1   2021-06-19
 

caf/impl/caf.c

@@ -341,7 +341,7 @@ void caf(Flower *flower, CactusParams *params, char *alignmentsFile, char *secon
         for (int64_t annealingRound = 0; annealingRound < annealingRoundsLength; annealingRound++) {
             int64_t minimumChainLength = annealingRounds[annealingRound];
             int64_t alignmentTrim = annealingRound < alignmentTrimLength ? alignmentTrims[annealingRound] : 0;
-            st_logDebug("Starting annealing round with a minimum chain length of %" PRIi64 " and an alignment trim of %" PRIi64 "\n", minimumChainLength, alignmentTrim);
+            st_logInfo("Starting annealing round with a minimum chain length of %" PRIi64 " and an alignment trim of %" PRIi64 "\n", minimumChainLength, alignmentTrim);
 
             stPinchIterator_setTrim(pinchIterator, alignmentTrim);
             if(secondaryPinchIterator != NULL) {
@@ -400,7 +400,7 @@ void caf(Flower *flower, CactusParams *params, char *alignmentsFile, char *secon
             //Do the melting rounds
             for (int64_t meltingRound = 0; meltingRound < meltingRoundsLength; meltingRound++) {
                 int64_t minimumChainLengthForMeltingRound = meltingRounds[meltingRound];
-                st_logDebug("Starting melting round with a minimum chain length of %" PRIi64 " \n", minimumChainLengthForMeltingRound);
+                st_logInfo("Starting melting round with a minimum chain length of %" PRIi64 " \n", minimumChainLengthForMeltingRound);
                 if (minimumChainLengthForMeltingRound >= minimumChainLength) {
                     break;
                 }

caf/impl/melting.c

@@ -130,7 +130,7 @@ void stCaf_melt(Flower *flower, stPinchThreadSet *threadSet, bool blockFilterfn(
                 0.0, breakChainsAtReverseTandems, maximumMedianSpacingBetweenLinkedEnds);
         stList *blocksToDelete = stCaf_getBlocksInChainsLessThanGivenLength(cactusGraph, minimumChainLength);
 
-        printf("A melting round is destroying %" PRIi64 " blocks with an average degree "
+        st_logInfo("A melting round is destroying %" PRIi64 " blocks with an average degree "
                "of %lf from chains with length less than %" PRIi64 ". Total aligned bases"
                " lost: %" PRIu64 "\n",
                stList_length(blocksToDelete), stCaf_averageBlockDegree(blocksToDelete),
@@ -488,8 +488,8 @@ void stCaf_meltRecoverableChains(Flower *flower, stPinchThreadSet *threadSet, bo
             }
         }
         int64_t numRecoverableBlocks = stList_length(blocksToDelete);
-        printf("Destroying %" PRIi64 " recoverable blocks\n", numRecoverableBlocks);
-        printf("The blocks covered %" PRIi64 " columns for a total of %" PRIi64 " aligned bases\n", numColumns(blocksToDelete), totalAlignedBases(blocksToDelete));
+        st_logInfo("Destroying %" PRIi64 " recoverable blocks\n", numRecoverableBlocks);
+        st_logInfo("The blocks covered %" PRIi64 " columns for a total of %" PRIi64 " aligned bases\n", numColumns(blocksToDelete), totalAlignedBases(blocksToDelete));
         stList_destruct(recoverableChains);
         stList_destruct(blocksToDelete);
 

src/cactus/cactus_progressive_config.xml

@@ -320,7 +320,7 @@
 	<graphmap_split
 		 minQueryCoverage="0.65"
 		 minQuerySmallCoverage="0.75"
-		 minQuerySmallThreshold="10000000"
+		 minQuerySmallThreshold="3000000"
 		 minQueryUniqueness="3"
 		 maxGap="50000"
 		 ambiguousName="_AMBIGUOUS_"

src/cactus/refmap/cactus_graphmap_join.py

@@ -204,7 +204,10 @@ def clip_vg(job, options, config, vg_path, vg_id):
     # todo: would be nice if clip-vg worked on stdin
     if options.normalizeIterations:
         normalized_path = vg_path + '.normalized'
-        cactus_call(parameters=['vg', 'mod', '-U', str(options.normalizeIterations), vg_path], outfile=normalized_path)
+        mod_cmd = ['vg', 'mod', '-U', str(options.normalizeIterations), vg_path]
+        if options.reference:
+            mod_cmd += ['-z', options.reference]
+        cactus_call(parameters=mod_cmd, outfile=normalized_path)
         # worth it
         cactus_call(parameters=['vg', 'validate', normalized_path])
         vg_path = normalized_path

src/cactus/shared/common.py

@@ -383,7 +383,7 @@ def getDockerImage():
 
 def getDockerRelease(gpu=False):
     """Get the most recent docker release."""
-    r = "quay.io/comparative-genomics-toolkit/cactus:v2.0.1"
+    r = "quay.io/comparative-genomics-toolkit/cactus:v2.0.2"
     if gpu:
         r += "-gpu"
     return r