Merge branch 'develop' into toppas_outdir

cbielow · Sep 20, 2024 · 024ea18 · 024ea18
2 parents bf534fa + 9e93970
commit 024ea18
Show file tree

Hide file tree

Showing 146 changed files with 2,292 additions and 597 deletions.
diff --git a/.github/workflows/knime_tests.yml b/.github/workflows/knime_tests.yml
@@ -1,13 +1,17 @@
 name: 'Test KNIME workflows'
 on:
   workflow_dispatch:
-
+    inputs:      
+      updateURL:
+        type: string
+        description: use a specific location for the knime update site
+        default: https://abibuilder.cs.uni-tuebingen.de/archive/openms/knime-plugin/updateSite/nightly/
 jobs:
   test-knime:
     env:
       GH_TOKEN: ${{ github.token }}
-      KNIME_MAJOR_VERSION: 4
-      KNIME_MINOR_VERSION: 7
+      KNIME_MAJOR_VERSION: 5
+      KNIME_MINOR_VERSION: 3
       INSTALLATION_DIR: ${{ github.workspace }}
     runs-on: ubuntu-latest
     steps:
@@ -37,7 +41,7 @@ jobs:
       - name: Install OpenMS plugin
         run: |
           "$KNIME_DIR/knime" -application org.eclipse.equinox.p2.director \
-            -r "http://update.knime.com/analytics-platform/${KNIME_VERSION},https://abibuilder.cs.uni-tuebingen.de/archive/openms/knime-plugin/updateSite/nightly/" \
+            -r "http://update.knime.com/analytics-platform/${KNIME_VERSION},${{ inputs.updateURL }}" \
             -p2.arch x86_64 \
             -profileProperties org.eclipse.update.install.features=true \
             -i "de.openms.feature.feature.group,com.genericworkflownodes.knime.feature.feature.group,de.openms.thirdparty.feature.feature.group" \

diff --git a/.github/workflows/openms_ci_matrix_full.yml b/.github/workflows/openms_ci_matrix_full.yml
@@ -174,8 +174,8 @@ jobs:
           fi
           echo "version_number=$VERSION_NUMBER" >> $GITHUB_OUTPUT
           grep -ne "----[[:space:]]*OpenMS" ${{ github.workspace }}/OpenMS/CHANGELOG > index_changelog.txt 
-          START=$(cat index_changelog.txt | grep -A 1 -e " $VERSION_NUMBER " | cut -f1 -d: | head -1)
-          END=$(cat index_changelog.txt | grep -A 1 -e " $VERSION_NUMBER " | cut -f1 -d: | tail -1)
+          START=$(cat index_changelog.txt | grep -A 1 -E " $VERSION_NUMBER(\.0)? " | cut -f1 -d: | head -1)
+          END=$(cat index_changelog.txt | grep -A 1 -E " $VERSION_NUMBER(\.0)? " | cut -f1 -d: | tail -1)
           echo "Extracting between lines:"
           echo $START
           echo $END
@@ -266,6 +266,8 @@ jobs:
         fi
 
         if [[ "${{ matrix.os }}" == macos-* ]];      then
+          ## Update the package lists for Brew
+          brew update
           ## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
           brew install python3 --force --overwrite
           brew install --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache
@@ -537,7 +539,7 @@ jobs:
       - name: Download source archive as artifact
         uses: actions/download-artifact@v4
         with:
-          name: OpenMS-${{ steps.create_changelog.outputs.version_number }}.tar.gz
+          name: OpenMS-${{ needs.build-and-test.outputs.version_number }}.tar.gz
 
       - name: Download changelog as artifact
         if: inputs.do_release
@@ -585,7 +587,7 @@ jobs:
           mkdir -p ~/.ssh/
           echo "$PASS" > ~/.ssh/private.key
           sudo chmod 600 ~/.ssh/private.key
-          ln -s ./$folder latest  #create link to the release folder
+          ln -s ../$folder latest  #create link to the release folder
           rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/OpenMSInstaller/release"
 
       - name: create RELEASE_TEXT
@@ -665,7 +667,7 @@ jobs:
           if [[ "${{ github.ref_name }}" == "nightly" ]]; then
             folder=nightly
           elif [[ "${{ github.ref_name }}" == release/* ]]; then
-            folder=release/${{ github.ref_name }}
+            folder=${{ github.ref_name }}
           else
             folder=experimental/${{ github.ref_name }}
           fi
@@ -696,15 +698,15 @@ jobs:
           echo "$PASS" > ~/.ssh/private.key
           sudo chmod 600 ~/.ssh/private.key
           ln -s ./$folder latest  #we can use the same link from above.
-          rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/Documentation/release
+          rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/Documentation/release"
       
         # TODO create softlinks to latest nightly
         # TODO create and upload file hashes, at least for release candidate
 
 
   build-deploy-knime-updatesite:
     env:
-      KNIME: 5.1
+      KNIME: 5.3
       JAVA_VER: 17
       PLUGIN_BUILD: ${{ github.workspace }}/plugin-build
       PLUGIN_SOURCE: ${{ github.workspace }}/plugin-source
@@ -871,7 +873,7 @@ jobs:
           if [[ "${{ github.ref_name }}" == "nightly" ]]; then
             folder=nightly
           elif [[ "${{ github.ref_name }}" == release/* ]]; then
-            folder=release/${{ github.ref_name }}
+            folder=${{ github.ref_name }}
           else
             folder=experimental/${{ github.ref_name }}
           fi
@@ -907,8 +909,8 @@ jobs:
           mkdir -p ~/.ssh/
           echo "$PASS" > ~/.ssh/private.key
           sudo chmod 600 ~/.ssh/private.key
-          ln -s ./$folder latest  #create link to the release folder
-          rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/knime-plugin/updateSite/release
+          ln -s ../$folder latest  #create link to the release folder
+          rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/knime-plugin/updateSite/release"
 
   do-release:
     if: inputs.do_release

diff --git a/.github/workflows/pyopenms-wheels.yml b/.github/workflows/pyopenms-wheels.yml
@@ -181,6 +181,8 @@ jobs:
 
     - name: Install contrib packages from brew
       run: |
+        ## Update the package lists for Brew
+        brew update
         ## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
         brew install python3 --force --overwrite
         brew install --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache
@@ -295,6 +297,8 @@ jobs:
 
     - name: Install contrib packages from brew
       run: |
+        ## Update the package lists for Brew
+        brew update
         ## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
         brew install python3 --force --overwrite
         brew install --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache

diff --git a/.github/workflows/update_version_numbers.yml b/.github/workflows/update_version_numbers.yml
@@ -16,7 +16,7 @@ jobs:
     runs-on: macos-latest
     steps:
       # Getting files (OpenMS)
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       # Update files with new package version numbers
       - name: update files
@@ -40,6 +40,9 @@ jobs:
 
           # update test write ini out:
           sed -i '' "s#<ITEM name=\"version\" value=\".*\" type=\"string\"#<ITEM name=\"version\" value=\"$package_version\" type=\"string\"#g" ./src/tests/topp/WRITE_INI_OUT.ini
+          # update INIUpdater version
+          sed -i '' "s#<ITEM name=\"version\" value=\".*\" type=\"string\"#<ITEM name=\"version\" value=\"$package_version\" type=\"string\"#g" ./src/tests/topp/INIUpdater_1_noupdate.toppas
+
 
           # update INIs in tests topp:
           find ./src/tests/topp/ -type f -name '*.ini' -exec grep -q "<ITEM name=\"version\" value=\".*\" type=\"string\"" {} \; -exec sed -i '' "s#<ITEM name=\"version\" value=\".*\" type=\"string\"#<ITEM name=\"version\" value=\"$package_version\" type=\"string\"#g" {} \;
@@ -76,7 +79,7 @@ jobs:
       # Commit and PR updated files
       - name: Commit and PR updated files
         if: env.changes_exist
-        uses: peter-evans/create-pull-request@v4
+        uses: peter-evans/create-pull-request@v7
         with:
           commit-message: 'Updated OpenMS package version number'
           committer: GitHub <noreply@github.com>

diff --git a/AUTHORS b/AUTHORS
@@ -75,6 +75,7 @@ the authors tag in the respective file header.
  - Martin Langwisch
  - Mateusz Łącki
  - Mathias Walzer
+ - Matteo Pilz
  - Matthew The
  - Matthias Bernt
  - Michael R. Crusoe
@@ -87,6 +88,7 @@ the authors tag in the respective file header.
  - Patricia Scheil
  - Peter Kunszt
  - Petra Gutenbrunner
+ - Radu Suciu
  - Ralf Gabriels
  - Rene Hussong
  - Ruben Grünberg
@@ -114,4 +116,4 @@ the authors tag in the respective file header.
  - Volker Mosthaf
  - Witold Wolski
  - Xiao Liang
- - Radu Suciu
+ - Yasset Perez-Riverol
diff --git a/CHANGELOG b/CHANGELOG
@@ -13,42 +13,61 @@ PR - Pull Request (on GitHub), i.e. integration of a new feature or bugfix
 #<number>, e.g. #4957 - a reference to an issue or pull request on GitHub, visit e.g. https://github.com/OpenMS/OpenMS/pull/XXXX (replace XXXX with number of interest) for details
 
 ------------------------------------------------------------------------------------------
-----                                OpenMS 3.2     (in development)                   ----
+----                                OpenMS 3.3.0     (under development)              ----
+------------------------------------------------------------------------------------------
+
+
+------------------------------------------------------------------------------------------
+----                                OpenMS 3.2.0                                      ----
 ------------------------------------------------------------------------------------------
 
 What's new:
-- Breaking change: Rename of parameters for FeatureFinderCentroided (debug->advanced), and PeakPickerWavelet/TOFCalibration (optimization -> optimization:type) (#7154)
+- Changes breaking backwards compatibility: 
+   - Rename of parameters for TOPP tool FeatureFinderCentroided (debug -> advanced), and PeakPickerWavelet/TOFCalibration (optimization -> optimization:type) (#7154)
+   - Rename of parameters for TOPP tool IDFilter (score:pep -> score:psm; score:prot -> score:protein; score:protgroup -> score:proteingroup) with 'nan' as new default (#7541)
+   - 3.2.0 KNIME package requires KNIME 5.3 or later
+- Support for SubsetNeighborSearch (SNS) via DecoyDatabase (#7565)
 
 Library:
 - Extend FileHandler to support load and store operations for our major datastructures (spectra, features, identifications, etc.). Replaced file type specific code with the more generic FileHandler calls to decouple the IO code from other parts of the library.
 - SiriusAdapter reworked to SiriusExport: Instead of running SIRIUS directly, this reworked tool takes multiple mzML and feautureXML (optional) files exporting a single SIRIUS .ms input file as well as an input table with compound info from features for the new AssayGeneratorMetaboSirius tool. (#7234)
 - Splitting AssayGeneratorMetabo into two tools: In line with the changes to SiriusExport this tool has been split into two separate workflows. AssayGeneratorMetabo generates an assay library from mzML and feautreXML files using an heuristic approach picking the highest intensity MS2 peaks (like before). AssayGeneratorMetaboSirius takes an existing SIRIUS project directory as input to generate an assay library based on fragmentation trees. (#7234)
 - better documentation for all SpectraFilter... tools (#7183)
-- TOPPView: offer Ion mobility view from 2D spectra view (#7423)
-- TOPPView: view ion mobility frames, irrespective of its MS level (formerly only MS1 was supported) (#7427)
+- TOPPView: 
+  - offer Ion mobility view from 2D spectra view (#7423)
+  - view ion mobility frames, irrespective of its MS level (formerly only MS1 was supported) (#7427)
 - OpenSwath: Add output on peak shape metrics to .osw file (#7222)
 
 New Tools:
 - IonMobilityBinning - Merges spectra with similar IM values and creates @p N output mzML's by discretizing the IM range (#7459)
+- AssayGeneratorMetaboSirius -- Assay library generation from a SIRIUS project directory (Metabolomics)
+- SiriusExport -- Metabolite identification using single and tandem mass spectrometry
 
 Fixes:
 - FileConverter: more robust (#7176)
 - MSFragger: allow relative path to database (#7155)
 - MSGFPlusAdapter: allow concurrent creation of indexed database (#7272)
+- CometAdapter: work around bug in Comet 2024.01 rev. 0 to avoid empty results (#7540)
 - ParamEditor: fixed error for the subsection parameter (ParamNode) to go through store function (#7180)
-- TOPPView: fix crash when viewing certain Chromatograms (#7220)
-- TOPPView: in 2D view, show correct adjacent layers in context menu, if user clicked to the right of the last MS1 scan (now shows the 4 rightmost MS1 scans, used to show the 4 leftmost scans) (#7423)
+- TOPPView:
+   - fix crash when viewing certain Chromatograms (#7220)
+   - in 2D view, show correct adjacent layers in context menu, if user clicked to the right of the last MS1 scan (now shows the 4 rightmost MS1 scans, used to show the 4 leftmost scans) (#7423)
+   - fix glitches in 1D view and layer names (#7549)
+   - Show prefix ions (e.g. b1) when generating theoretical spectra (#7567)
 - TOPPAS: open files in TOPPView (#7213)
 - pyOpenMS: Log warnings in pure Python code with warnings.warn instead of print (#7418)
 - more robust parsing of mzIdentML (#7153)
+- OpenSwath: Fix bug in diaPASEF window determination (#7546)  
 
 Misc:
-- FileInfo: Report IM ranges (if any) (#7459)
+- FileInfo: Report ion mobility ranges (if any) (#7459)
 - OpenMSInfo reports the ILP solver (CoinOr or glpk) (#7156)
 - add citation information for OpenMS 3.0 (Nat. Methods) (#7383)
 - Add export for Common Workflow Language (CWL) (#6156)
 - Add tool description lib (TDL) dependency (#6156)
 
+Changed/new parameters: see CHANGELOG_PARAMS for details
+
 Note: The goal of our library is to provide useful, reusable code in a way that’s easy to understand and use.
 To make OpenMS simpler, more focused and more accessible we gradually remove potentially outdated tools and algorithms.
 If you, as a user, are negatively affected by this step please contact us. We listen to our users and will try to find 
@@ -57,14 +76,20 @@ an alternative solution or reverse a particular decision for removal.
 Cleanup/Removal:
 
 - removed tools:
-  - Removed SpectraFilterMarkerMower TOPP tool (#7183)
-  - FeatureFinderIsotopeWavelet
-  - PeakPickerWavelet
-  - PrecursorMassCorrector
-  - TOFCalibration
-  - ERPairFinder
-  - RNPxlSearch
-  - SpectraFilterParentPeakMower
+  - SpectraFilterMarkerMower -- Applies thresholdfilter to peak spectra (#7183)
+  - FeatureFinderIsotopeWavelet -- Detects two-dimensional features in LC-MS data
+  - PeakPickerWavelet -- Finds mass spectrometric peaks in profile mass spectra
+  - PrecursorMassCorrector -- Corrects the precursor entries of MS/MS spectra, by using MS1 information
+  - TOFCalibration -- Applies time of flight calibration
+  - ERPairFinder -- Util which can be used to evaluate pair ratios on enhanced resolution (zoom) scans
+  - RNPxlSearch -- Annotate RNA/DNA-peptide cross-links in MS/MS spectra
+  - SpectraFilterSqrtMower -- Applies thresholdfilter to peak spectra
+  - FeatureFinderMRM -- Detects two-dimensional features in LC-MS data
+  - MapAlignerSpectrum -- Corrects retention time distortions between maps by spectrum alignment
+  - ProteinResolver -- protein inference
+  - SiriusAdapter -- Tool for metabolite identification using single and tandem mass spectrometry
+  - SpectraFilterBernNorm -- Applies thresholdfilter to peak spectra
+  - SpectraFilterScaler -- Applies thresholdfilter to peak spectra
 
 - removed library code:
   - HiddenMarkovModel
@@ -87,6 +112,9 @@ Cleanup/Removal:
   - PeakPickerCWT_test
   - PeakShape_test
 
+- Deprecated (and likely removed in the next release)
+  - XTandemAdapter
+
 ------------------------------------------------------------------------------------------
 ----                                OpenMS 3.1     (released 10/2023)                 ----
 ------------------------------------------------------------------------------------------