Merge branch 'develop' into qt6

.github/workflows/knime_tests.yml

@@ -1,13 +1,17 @@
 name: 'Test KNIME workflows'
 on:
   workflow_dispatch:
-
+    inputs:      
+      updateURL:
+        type: string
+        description: use a specific location for the knime update site
+        default: https://abibuilder.cs.uni-tuebingen.de/archive/openms/knime-plugin/updateSite/nightly/
 jobs:
   test-knime:
     env:
       GH_TOKEN: ${{ github.token }}
-      KNIME_MAJOR_VERSION: 4
-      KNIME_MINOR_VERSION: 7
+      KNIME_MAJOR_VERSION: 5
+      KNIME_MINOR_VERSION: 3
       INSTALLATION_DIR: ${{ github.workspace }}
     runs-on: ubuntu-latest
     steps:
@@ -37,7 +41,7 @@ jobs:
       - name: Install OpenMS plugin
         run: |
           "$KNIME_DIR/knime" -application org.eclipse.equinox.p2.director \
-            -r "http://update.knime.com/analytics-platform/${KNIME_VERSION},https://abibuilder.cs.uni-tuebingen.de/archive/openms/knime-plugin/updateSite/nightly/" \
+            -r "http://update.knime.com/analytics-platform/${KNIME_VERSION},${{ inputs.updateURL }}" \
             -p2.arch x86_64 \
             -profileProperties org.eclipse.update.install.features=true \
             -i "de.openms.feature.feature.group,com.genericworkflownodes.knime.feature.feature.group,de.openms.thirdparty.feature.feature.group" \

.github/workflows/openms_ci_matrix_full.yml

@@ -174,8 +174,8 @@ jobs:
           fi
           echo "version_number=$VERSION_NUMBER" >> $GITHUB_OUTPUT
           grep -ne "----[[:space:]]*OpenMS" ${{ github.workspace }}/OpenMS/CHANGELOG > index_changelog.txt 
-          START=$(cat index_changelog.txt | grep -A 1 -e " $VERSION_NUMBER " | cut -f1 -d: | head -1)
-          END=$(cat index_changelog.txt | grep -A 1 -e " $VERSION_NUMBER " | cut -f1 -d: | tail -1)
+          START=$(cat index_changelog.txt | grep -A 1 -E " $VERSION_NUMBER(\.0)? " | cut -f1 -d: | head -1)
+          END=$(cat index_changelog.txt | grep -A 1 -E " $VERSION_NUMBER(\.0)? " | cut -f1 -d: | tail -1)
           echo "Extracting between lines:"
           echo $START
           echo $END
@@ -265,8 +265,10 @@ jobs:
         fi
 
         if [[ "${{ matrix.os }}" == macos-* ]];      then
+          ## Update the package lists for Brew
+          brew update
           ## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
-          brew install python3 --force --overwrite
+          brew install python@3.12 --force --overwrite
           brew install --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache
           brew install libsvm xerces-c boost eigen sqlite coinutils cbc cgl clp qt
           echo "cmake_prefix=$(brew --prefix qt)/lib/cmake;$(brew --prefix qt)" >> $GITHUB_OUTPUT
@@ -454,7 +456,7 @@ jobs:
         type: 'tar'
         directory: ${{ github.workspace }}
         path: OpenMS
-        exclusions: 'bld/* ./OpenMS-${{ steps.create_changelog.outputs.version_number }}.tar.gz THIRDPARTY/* .git/*'
+        exclusions: 'bld/* ./OpenMS-${{ steps.create_changelog.outputs.version_number }}.tar.gz ./THIRDPARTY/** .git/*'
         filename: 'OpenMS-${{ steps.create_changelog.outputs.version_number }}.tar.gz'
 
 
@@ -535,7 +537,7 @@ jobs:
       - name: Download source archive as artifact
         uses: actions/download-artifact@v4
         with:
-          name: OpenMS-${{ steps.create_changelog.outputs.version_number }}.tar.gz
+          name: OpenMS-${{ needs.build-and-test.outputs.version_number }}.tar.gz
 
       - name: Download changelog as artifact
         if: inputs.do_release
@@ -583,7 +585,7 @@ jobs:
           mkdir -p ~/.ssh/
           echo "$PASS" > ~/.ssh/private.key
           sudo chmod 600 ~/.ssh/private.key
-          ln -s ./$folder latest  #create link to the release folder
+          ln -s ../$folder latest  #create link to the release folder
           rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/OpenMSInstaller/release"
 
       - name: create RELEASE_TEXT
@@ -663,7 +665,7 @@ jobs:
           if [[ "${{ github.ref_name }}" == "nightly" ]]; then
             folder=nightly
           elif [[ "${{ github.ref_name }}" == release/* ]]; then
-            folder=release/${{ github.ref_name }}
+            folder=${{ github.ref_name }}
           else
             folder=experimental/${{ github.ref_name }}
           fi
@@ -694,15 +696,15 @@ jobs:
           echo "$PASS" > ~/.ssh/private.key
           sudo chmod 600 ~/.ssh/private.key
           ln -s ./$folder latest  #we can use the same link from above.
-          rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/Documentation/release
+          rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/Documentation/release"
       
         # TODO create softlinks to latest nightly
         # TODO create and upload file hashes, at least for release candidate
 
 
   build-deploy-knime-updatesite:
     env:
-      KNIME: 5.1
+      KNIME: 5.3
       JAVA_VER: 17
       PLUGIN_BUILD: ${{ github.workspace }}/plugin-build
       PLUGIN_SOURCE: ${{ github.workspace }}/plugin-source
@@ -869,7 +871,7 @@ jobs:
           if [[ "${{ github.ref_name }}" == "nightly" ]]; then
             folder=nightly
           elif [[ "${{ github.ref_name }}" == release/* ]]; then
-            folder=release/${{ github.ref_name }}
+            folder=${{ github.ref_name }}
           else
             folder=experimental/${{ github.ref_name }}
           fi
@@ -906,7 +908,7 @@ jobs:
           echo "$PASS" > ~/.ssh/private.key
           sudo chmod 600 ~/.ssh/private.key
           ln -s ./$folder latest  #create link to the release folder
-          rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/knime-plugin/updateSite/release
+          rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/knime-plugin/updateSite/release"
 
   do-release:
     if: inputs.do_release

.github/workflows/pyopenms-wheels.yml

@@ -135,7 +135,7 @@ jobs:
           conda remove --name "pyoms-bld-${pynodot}" --all
 
           # clean previous pyopenms libs
-          find . -name "pyopenms*.so" -exec rm -rf {} \;
+          find . -name "_pyopenms*.pyd" -exec rm -rf {} \;
 
         done
 
@@ -181,8 +181,10 @@ jobs:
 
     - name: Install contrib packages from brew
       run: |
+        ## Update the package lists for Brew
+        brew update
         ## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
-        brew install python3 --force --overwrite
+        brew install python@3.12 --force --overwrite
         brew install --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache
         brew install libsvm xerces-c boost eigen sqlite coinutils cbc cgl clp qt@5 libomp
         echo "cmake_prefix=$(brew --prefix qt@5)/lib/cmake;$(brew --prefix qt@5)" >> $GITHUB_OUTPUT
@@ -240,7 +242,7 @@ jobs:
           conda remove --name pyoms-bld-"${pynodot}" --all
 
           # clean previous pyopenms libs
-          find . -name "pyopenms*.so" -exec rm -rf {} \;
+          find . -name "_pyopenms*.so" -exec rm -rf {} \;
 
         done
 
@@ -295,8 +297,10 @@ jobs:
 
     - name: Install contrib packages from brew
       run: |
+        ## Update the package lists for Brew
+        brew update
         ## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
-        brew install python3 --force --overwrite
+        brew install python@3.12 --force --overwrite
         brew install --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache
         brew install libsvm xerces-c boost eigen sqlite coinutils cbc cgl clp qt@5 libomp
         echo "cmake_prefix=$(brew --prefix qt@5)/lib/cmake;$(brew --prefix qt@5)" >> $GITHUB_OUTPUT
@@ -354,7 +358,7 @@ jobs:
           conda remove --name pyoms-bld-"${pynodot}" --all
 
           # clean previous pyopenms libs
-          find . -name "pyopenms*.so" -exec rm -rf {} \;
+          find . -name "_pyopenms*.so" -exec rm -rf {} \;
 
         done
         
@@ -592,6 +596,6 @@ jobs:
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         user: __token__
-        repository-url: https://test.pypi.org/
+        repository-url: https://pypi.org/
         password: ${{ secrets.pypi_api_token_release }}
         packages-dir: ${{ github.workspace }}/wheels

.github/workflows/update_version_numbers.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: macos-latest
     steps:
       # Getting files (OpenMS)
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       # Update files with new package version numbers
       - name: update files
@@ -40,6 +40,9 @@ jobs:
 
           # update test write ini out:
           sed -i '' "s#<ITEM name=\"version\" value=\".*\" type=\"string\"#<ITEM name=\"version\" value=\"$package_version\" type=\"string\"#g" ./src/tests/topp/WRITE_INI_OUT.ini
+          # update INIUpdater version
+          sed -i '' "s#<ITEM name=\"version\" value=\".*\" type=\"string\"#<ITEM name=\"version\" value=\"$package_version\" type=\"string\"#g" ./src/tests/topp/INIUpdater_1_noupdate.toppas
+
 
           # update INIs in tests topp:
           find ./src/tests/topp/ -type f -name '*.ini' -exec grep -q "<ITEM name=\"version\" value=\".*\" type=\"string\"" {} \; -exec sed -i '' "s#<ITEM name=\"version\" value=\".*\" type=\"string\"#<ITEM name=\"version\" value=\"$package_version\" type=\"string\"#g" {} \;
@@ -76,7 +79,7 @@ jobs:
       # Commit and PR updated files
       - name: Commit and PR updated files
         if: env.changes_exist
-        uses: peter-evans/create-pull-request@v4
+        uses: peter-evans/create-pull-request@v7
         with:
           commit-message: 'Updated OpenMS package version number'
           committer: GitHub <noreply@github.com>

AUTHORS

@@ -53,6 +53,7 @@ the authors tag in the respective file header.
  - Johan Teleman
  - Johannes Junker
  - Johannes Veit
+ - Johannes von Kleist
  - Joshua Charkow
  - Julia Thueringer
  - Juliane Schmachtenberg
@@ -91,6 +92,7 @@ the authors tag in the respective file header.
  - Radu Suciu
  - Ralf Gabriels
  - Rene Hussong
+ - Rick Helmus
  - Ruben Grünberg
  - Samuel Wein
  - Sandro Andreotti

CHANGELOG

@@ -1,6 +1,4 @@
 
-
-
 Glossary:
 OpenMS - Name of the project and our C++ library
 TOPP - "The OpenMS PiPeline", collection of chainable tools for flexible HPLC/MS workflows. Formerly known as "The OpenMS Proteomics Pipeline"
@@ -13,42 +11,69 @@ PR - Pull Request (on GitHub), i.e. integration of a new feature or bugfix
 #<number>, e.g. #4957 - a reference to an issue or pull request on GitHub, visit e.g. https://github.com/OpenMS/OpenMS/pull/XXXX (replace XXXX with number of interest) for details
 
 ------------------------------------------------------------------------------------------
-----                                OpenMS 3.2.0     (under development)              ----
+----                                OpenMS 3.3.0     (under development)              ----
+------------------------------------------------------------------------------------------
+
+What's new:
+- Changes breaking backwards compatibility: 
+  - the QualityControl TOPP tool has some renamed parameters and supports output directories (#7497)
+
+Misc:
+- FileInfo:
+    - support MzTab files (#7568)
+- TOPPAS 
+    - supports writing results to output directories  (for TOPP tools which have such parameters) (#7497)
+    - TOPPAS tutorial enhanced (#7497)
+- FeatureFinderMetabo
+    - added report_smoothed_intensities parameter (#7594)
+
+------------------------------------------------------------------------------------------
+----                                OpenMS 3.2.0       (released 09/2024)             ----
 ------------------------------------------------------------------------------------------
 
 What's new:
 - Changes breaking backwards compatibility: 
    - Rename of parameters for TOPP tool FeatureFinderCentroided (debug -> advanced), and PeakPickerWavelet/TOFCalibration (optimization -> optimization:type) (#7154)
    - Rename of parameters for TOPP tool IDFilter (score:pep -> score:psm; score:prot -> score:protein; score:protgroup -> score:proteingroup) with 'nan' as new default (#7541)
+   - 3.2.0 KNIME package requires KNIME 5.3 or later
+- Support for SubsetNeighborSearch (SNS) via DecoyDatabase (#7565)
+- SageAdapter received large updates including added functionality for PTM discovery + enabling features such as chimera seach, RT prediction, filtering by q-value, etc. 
 
 Library:
 - Extend FileHandler to support load and store operations for our major datastructures (spectra, features, identifications, etc.). Replaced file type specific code with the more generic FileHandler calls to decouple the IO code from other parts of the library.
 - SiriusAdapter reworked to SiriusExport: Instead of running SIRIUS directly, this reworked tool takes multiple mzML and feautureXML (optional) files exporting a single SIRIUS .ms input file as well as an input table with compound info from features for the new AssayGeneratorMetaboSirius tool. (#7234)
 - Splitting AssayGeneratorMetabo into two tools: In line with the changes to SiriusExport this tool has been split into two separate workflows. AssayGeneratorMetabo generates an assay library from mzML and feautreXML files using an heuristic approach picking the highest intensity MS2 peaks (like before). AssayGeneratorMetaboSirius takes an existing SIRIUS project directory as input to generate an assay library based on fragmentation trees. (#7234)
 - better documentation for all SpectraFilter... tools (#7183)
-- TOPPView: offer Ion mobility view from 2D spectra view (#7423)
-- TOPPView: view ion mobility frames, irrespective of its MS level (formerly only MS1 was supported) (#7427)
+- TOPPView: 
+  - offer Ion mobility view from 2D spectra view (#7423)
+  - view ion mobility frames, irrespective of its MS level (formerly only MS1 was supported) (#7427)
 - OpenSwath: Add output on peak shape metrics to .osw file (#7222)
 
 New Tools:
 - IonMobilityBinning - Merges spectra with similar IM values and creates @p N output mzML's by discretizing the IM range (#7459)
 - AssayGeneratorMetaboSirius -- Assay library generation from a SIRIUS project directory (Metabolomics)
 - SiriusExport -- Metabolite identification using single and tandem mass spectrometry
 
+
 Fixes:
 - FileConverter: more robust (#7176)
 - MSFragger: allow relative path to database (#7155)
 - MSGFPlusAdapter: allow concurrent creation of indexed database (#7272)
 - CometAdapter: work around bug in Comet 2024.01 rev. 0 to avoid empty results (#7540)
 - ParamEditor: fixed error for the subsection parameter (ParamNode) to go through store function (#7180)
-- TOPPView: fix crash when viewing certain Chromatograms (#7220)
-- TOPPView: in 2D view, show correct adjacent layers in context menu, if user clicked to the right of the last MS1 scan (now shows the 4 rightmost MS1 scans, used to show the 4 leftmost scans) (#7423)
+- TOPPView:
+   - fix crash when viewing certain Chromatograms (#7220)
+   - in 2D view, show correct adjacent layers in context menu, if user clicked to the right of the last MS1 scan (now shows the 4 rightmost MS1 scans, used to show the 4 leftmost scans) (#7423)
+   - fix glitches in 1D view and layer names (#7549)
+   - Show prefix ions (e.g. b1) when generating theoretical spectra (#7567)
 - TOPPAS: open files in TOPPView (#7213)
 - pyOpenMS: Log warnings in pure Python code with warnings.warn instead of print (#7418)
 - more robust parsing of mzIdentML (#7153)
+- SageAdapter now works with sage v0.15.0 and beyond
+- OpenSwath: Fix bug in diaPASEF window determination (#7546)  
 
 Misc:
-- FileInfo: Report IM ranges (if any) (#7459)
+- FileInfo: Report ion mobility ranges (if any) (#7459)
 - OpenMSInfo reports the ILP solver (CoinOr or glpk) (#7156)
 - add citation information for OpenMS 3.0 (Nat. Methods) (#7383)
 - Add export for Common Workflow Language (CWL) (#6156)

CMakeLists.txt

@@ -105,7 +105,7 @@ list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Windows")
 #------------------------------------------------------------------------------
 
 set(OPENMS_PACKAGE_VERSION_MAJOR "3")
-set(OPENMS_PACKAGE_VERSION_MINOR "2")
+set(OPENMS_PACKAGE_VERSION_MINOR "3")
 set(OPENMS_PACKAGE_VERSION_PATCH "0")
 set(OPENMS_PACKAGE_VERSION "${OPENMS_PACKAGE_VERSION_MAJOR}.${OPENMS_PACKAGE_VERSION_MINOR}.${OPENMS_PACKAGE_VERSION_PATCH}")
 

cmake/OpenMSConfig.cmake.in

@@ -50,6 +50,7 @@ endif()
 #TODO somehow add the same/compatible versions that were found by OpenMS? And what about static vs dynamic? E.g. if we link to static zlib in OpenMS(.dll) what (if at all) can/should the consumer link against?
 find_dependency(Qt6 @QT_MIN_VERSION@ COMPONENTS @OpenMS_QT_COMPONENTS@)
 find_dependency(XercesC)
+find_dependency(Eigen3 3.3.4)
 find_dependency(LIBSVM 2.91)
 
 # Rest are private linked libraries
@@ -58,7 +59,6 @@ find_dependency(LIBSVM 2.91)
 #find_dependency(GLPK)
 #find_dependency(ZLIB)
 #find_dependency(BZip2)
-#find_dependency(Eigen3 3.3.4)
 #find_dependency(SQLite3 3.15.0)
 #find_dependency(HDF5)