Merge branch 'develop' into mztab_fileinfo

.github/workflows/knime_tests.yml

@@ -1,13 +1,17 @@
 name: 'Test KNIME workflows'
 on:
   workflow_dispatch:
-
+    inputs:      
+      updateURL:
+        type: string
+        description: use a specific location for the knime update site
+        default: https://abibuilder.cs.uni-tuebingen.de/archive/openms/knime-plugin/updateSite/nightly/
 jobs:
   test-knime:
     env:
       GH_TOKEN: ${{ github.token }}
-      KNIME_MAJOR_VERSION: 4
-      KNIME_MINOR_VERSION: 7
+      KNIME_MAJOR_VERSION: 5
+      KNIME_MINOR_VERSION: 3
       INSTALLATION_DIR: ${{ github.workspace }}
     runs-on: ubuntu-latest
     steps:
@@ -37,7 +41,7 @@ jobs:
       - name: Install OpenMS plugin
         run: |
           "$KNIME_DIR/knime" -application org.eclipse.equinox.p2.director \
-            -r "http://update.knime.com/analytics-platform/${KNIME_VERSION},https://abibuilder.cs.uni-tuebingen.de/archive/openms/knime-plugin/updateSite/nightly/" \
+            -r "http://update.knime.com/analytics-platform/${KNIME_VERSION},${{ inputs.updateURL }}" \
             -p2.arch x86_64 \
             -profileProperties org.eclipse.update.install.features=true \
             -i "de.openms.feature.feature.group,com.genericworkflownodes.knime.feature.feature.group,de.openms.thirdparty.feature.feature.group" \

.github/workflows/openms_ci_matrix_full.yml

@@ -174,8 +174,8 @@ jobs:
           fi
           echo "version_number=$VERSION_NUMBER" >> $GITHUB_OUTPUT
           grep -ne "----[[:space:]]*OpenMS" ${{ github.workspace }}/OpenMS/CHANGELOG > index_changelog.txt 
-          START=$(cat index_changelog.txt | grep -A 1 -e " $VERSION_NUMBER " | cut -f1 -d: | head -1)
-          END=$(cat index_changelog.txt | grep -A 1 -e " $VERSION_NUMBER " | cut -f1 -d: | tail -1)
+          START=$(cat index_changelog.txt | grep -A 1 -E " $VERSION_NUMBER(\.0)? " | cut -f1 -d: | head -1)
+          END=$(cat index_changelog.txt | grep -A 1 -E " $VERSION_NUMBER(\.0)? " | cut -f1 -d: | tail -1)
           echo "Extracting between lines:"
           echo $START
           echo $END
@@ -266,6 +266,8 @@ jobs:
         fi
 
         if [[ "${{ matrix.os }}" == macos-* ]];      then
+          ## Update the package lists for Brew
+          brew update
           ## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
           brew install python3 --force --overwrite
           brew install --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache
@@ -537,7 +539,7 @@ jobs:
       - name: Download source archive as artifact
         uses: actions/download-artifact@v4
         with:
-          name: OpenMS-${{ steps.create_changelog.outputs.version_number }}.tar.gz
+          name: OpenMS-${{ needs.build-and-test.outputs.version_number }}.tar.gz
 
       - name: Download changelog as artifact
         if: inputs.do_release
@@ -585,7 +587,7 @@ jobs:
           mkdir -p ~/.ssh/
           echo "$PASS" > ~/.ssh/private.key
           sudo chmod 600 ~/.ssh/private.key
-          ln -s ./$folder latest  #create link to the release folder
+          ln -s ../$folder latest  #create link to the release folder
           rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/OpenMSInstaller/release"
 
       - name: create RELEASE_TEXT
@@ -665,7 +667,7 @@ jobs:
           if [[ "${{ github.ref_name }}" == "nightly" ]]; then
             folder=nightly
           elif [[ "${{ github.ref_name }}" == release/* ]]; then
-            folder=release/${{ github.ref_name }}
+            folder=${{ github.ref_name }}
           else
             folder=experimental/${{ github.ref_name }}
           fi
@@ -696,7 +698,7 @@ jobs:
           echo "$PASS" > ~/.ssh/private.key
           sudo chmod 600 ~/.ssh/private.key
           ln -s ./$folder latest  #we can use the same link from above.
-          rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/Documentation/release
+          rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/Documentation/release"
       
         # TODO create softlinks to latest nightly
         # TODO create and upload file hashes, at least for release candidate
@@ -871,7 +873,7 @@ jobs:
           if [[ "${{ github.ref_name }}" == "nightly" ]]; then
             folder=nightly
           elif [[ "${{ github.ref_name }}" == release/* ]]; then
-            folder=release/${{ github.ref_name }}
+            folder=${{ github.ref_name }}
           else
             folder=experimental/${{ github.ref_name }}
           fi
@@ -907,8 +909,8 @@ jobs:
           mkdir -p ~/.ssh/
           echo "$PASS" > ~/.ssh/private.key
           sudo chmod 600 ~/.ssh/private.key
-          ln -s ./$folder latest  #create link to the release folder
-          rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/knime-plugin/updateSite/release
+          ln -s ../$folder latest  #create link to the release folder
+          rsync --progress -avz -e "ssh -i ~/.ssh/private.key -o StrictHostKeyChecking=no" latest "$USER@$HOST:/knime-plugin/updateSite/release"
 
   do-release:
     if: inputs.do_release

.github/workflows/pyopenms-wheels.yml

@@ -181,6 +181,8 @@ jobs:
 
     - name: Install contrib packages from brew
       run: |
+        ## Update the package lists for Brew
+        brew update
         ## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
         brew install python3 --force --overwrite
         brew install --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache
@@ -295,6 +297,8 @@ jobs:
 
     - name: Install contrib packages from brew
       run: |
+        ## Update the package lists for Brew
+        brew update
         ## Needed for Qt. Install before to overwrite the default softlinks on the GH runners
         brew install python3 --force --overwrite
         brew install --quiet ccache autoconf automake libtool ninja && brew link --overwrite ccache

.github/workflows/update_version_numbers.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: macos-latest
     steps:
       # Getting files (OpenMS)
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       # Update files with new package version numbers
       - name: update files
@@ -40,6 +40,9 @@ jobs:
 
           # update test write ini out:
           sed -i '' "s#<ITEM name=\"version\" value=\".*\" type=\"string\"#<ITEM name=\"version\" value=\"$package_version\" type=\"string\"#g" ./src/tests/topp/WRITE_INI_OUT.ini
+          # update INIUpdater version
+          sed -i '' "s#<ITEM name=\"version\" value=\".*\" type=\"string\"#<ITEM name=\"version\" value=\"$package_version\" type=\"string\"#g" ./src/tests/topp/INIUpdater_1_noupdate.toppas
+
 
           # update INIs in tests topp:
           find ./src/tests/topp/ -type f -name '*.ini' -exec grep -q "<ITEM name=\"version\" value=\".*\" type=\"string\"" {} \; -exec sed -i '' "s#<ITEM name=\"version\" value=\".*\" type=\"string\"#<ITEM name=\"version\" value=\"$package_version\" type=\"string\"#g" {} \;
@@ -76,7 +79,7 @@ jobs:
       # Commit and PR updated files
       - name: Commit and PR updated files
         if: env.changes_exist
-        uses: peter-evans/create-pull-request@v4
+        uses: peter-evans/create-pull-request@v7
         with:
           commit-message: 'Updated OpenMS package version number'
           committer: GitHub <noreply@github.com>

AUTHORS

@@ -53,6 +53,7 @@ the authors tag in the respective file header.
  - Johan Teleman
  - Johannes Junker
  - Johannes Veit
+ - Johannes von Kleist
  - Joshua Charkow
  - Julia Thueringer
  - Juliane Schmachtenberg

CHANGELOG

@@ -13,13 +13,26 @@ PR - Pull Request (on GitHub), i.e. integration of a new feature or bugfix
 #<number>, e.g. #4957 - a reference to an issue or pull request on GitHub, visit e.g. https://github.com/OpenMS/OpenMS/pull/XXXX (replace XXXX with number of interest) for details
 
 ------------------------------------------------------------------------------------------
-----                                OpenMS 3.2.0     (under development)              ----
+----                                OpenMS 3.3.0     (under development)              ----
 ------------------------------------------------------------------------------------------
 
+- FileInfo:
+    - support MzTab files (#7568)
+
+
+------------------------------------------------------------------------------------------
+----                                OpenMS 3.2.0                                      ----
+------------------------------------------------------------------------------------------
+
+
+
 What's new:
 - Changes breaking backwards compatibility: 
    - Rename of parameters for TOPP tool FeatureFinderCentroided (debug -> advanced), and PeakPickerWavelet/TOFCalibration (optimization -> optimization:type) (#7154)
    - Rename of parameters for TOPP tool IDFilter (score:pep -> score:psm; score:prot -> score:protein; score:protgroup -> score:proteingroup) with 'nan' as new default (#7541)
+   - 3.2.0 KNIME package requires KNIME 5.3 or later
+- Support for SubsetNeighborSearch (SNS) via DecoyDatabase (#7565)
+- SageAdapter received large updates including added functionality for PTM discovery + enabling features such as chimera seach, RT prediction, filtering by q-value, etc. 
 
 Library:
 - Extend FileHandler to support load and store operations for our major datastructures (spectra, features, identifications, etc.). Replaced file type specific code with the more generic FileHandler calls to decouple the IO code from other parts of the library.
@@ -36,6 +49,7 @@ New Tools:
 - AssayGeneratorMetaboSirius -- Assay library generation from a SIRIUS project directory (Metabolomics)
 - SiriusExport -- Metabolite identification using single and tandem mass spectrometry
 
+
 Fixes:
 - FileConverter: more robust (#7176)
 - MSFragger: allow relative path to database (#7155)
@@ -50,11 +64,11 @@ Fixes:
 - TOPPAS: open files in TOPPView (#7213)
 - pyOpenMS: Log warnings in pure Python code with warnings.warn instead of print (#7418)
 - more robust parsing of mzIdentML (#7153)
+- SageAdapter now works with sage v0.15.0 and beyond
+- OpenSwath: Fix bug in diaPASEF window determination (#7546)  
 
 Misc:
-- FileInfo:
-    - Report IM ranges (if any) (#7459)
-    - support Mztab files (#7568)
+- FileInfo: Report ion mobility ranges (if any) (#7459)
 - OpenMSInfo reports the ILP solver (CoinOr or glpk) (#7156)
 - add citation information for OpenMS 3.0 (Nat. Methods) (#7383)
 - Add export for Common Workflow Language (CWL) (#6156)

CMakeLists.txt

@@ -105,7 +105,7 @@ list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Windows")
 #------------------------------------------------------------------------------
 
 set(OPENMS_PACKAGE_VERSION_MAJOR "3")
-set(OPENMS_PACKAGE_VERSION_MINOR "2")
+set(OPENMS_PACKAGE_VERSION_MINOR "3")
 set(OPENMS_PACKAGE_VERSION_PATCH "0")
 set(OPENMS_PACKAGE_VERSION "${OPENMS_PACKAGE_VERSION_MAJOR}.${OPENMS_PACKAGE_VERSION_MINOR}.${OPENMS_PACKAGE_VERSION_PATCH}")
 

doc/doxygen/public/TOPPAS.doxygen

@@ -46,12 +46,18 @@
 		@image html TOPPAS_simple_example.png
 		@image latex TOPPAS_simple_example.png "" width=14cm
 
-
-		To create a new TOPPAS file, you can either: 
+    You can play with existing @em example @em pipelines, which cover identification, 
+    quantification and some special use cases, such as SubsetNeighborSearch.
+    To open an existing example pipeline, select @p File > @p Open @p example @p file from the menu bar in TOPPAS.
+    See @ref TOPPAS_examples below for details.
+
+		To create a new (empty) TOPPAS file, you can either: 
 
 			- open TOPPAS without providing any existing workflow - an empty workflow will be opened automatically
 			- in a running TOPPAS program choose: @p File @p > @p New
-			- create an empty file in your file browser (explorer) with the suffix @p \.toppas and double-click it (on Windows systems all @p \.toppas files are associated with TOPPAS automatically during installation of %OpenMS, on Linux and MacOS you might need to manually associate the extension)
+			- create an empty file with the suffix @p \.toppas 
+
+    All @p \.toppas files are usually associated with TOPPAS automatically during installation of %OpenMS - at least on Windows. On Linux and MacOS you might need to manually associate the \.toppas extension.
 
 	@page TOPPAS_interface User interface
 
@@ -246,26 +252,30 @@
 
 	@page TOPPAS_examples Examples
 
-		The following sections explain the example pipelines TOPPAS comes with. You can
-		open them by selecting @p File > @p Open @p example @p file. All input files and
-		parameters are already specified, so you can just hit @p Pipeline > @p Run (or press
+		The following sections explain the example pipelines TOPPAS comes with.
+
+    You can @em open all examples pipelines by selecting @p File > @p Open @p example @p file in TOPPAS.
+
+    All input files and parameters are already specified, so you can just hit @p Pipeline > @p Run (or press
 		@p F5) and see what happens.
 
 		@section TOPPAS_peak_picking_example Profile data processing
 
-		The file @p peakpicker_tutorial.toppas contains a simple pipeline representing a
+		The file @p peakpicker_tutorial.toppas can be inspect it in TOPPAS via `File -> Open Example File`.
+    It contains a simple pipeline representing a
 		common use case: starting with profile data, the noise is eliminated and the baseline
 		is subtracted. Then, PeakPickerHiRes is used to find all peaks in the noise-filtered
 		and baseline-reduced profile data. 
+
 
 		@image html TOPPAS_example_profile_data_processing.png
 		@image latex TOPPAS_example_profile_data_processing.png "" width=14cm
 
 		@section TOPPAS_id_example Identification of E. coli peptides
 
 		This section describes an example identification pipeline contained in the 
-		example directory, @p Ecoli_Identification.toppas. It is shipped together
-		with a reduced example mzML file containing 139 MS2 spectra from an E. coli
+		example directory, @p Ecoli_Identification.toppas.  Inspect it in TOPPAS via `File -> Open Example File`.
+    It is shipped together with a reduced example mzML file containing 139 MS2 spectra from an E. coli
 		run on an Orbitrap instrument as well as an E. coli target-decoy database.
 
 		We use the search engine
@@ -293,7 +303,8 @@
 		@section TOPPAS_quant_example Quantitation of BSA runs
 
 		The simple pipeline described in this section (@p BSA_Quantitation.toppas) can be used to quantify peptides
-		that occur on different runs. The example dataset contains three different bovine serum albumin (BSA) runs.
+		that occur on different runs. Inspect it in TOPPAS via `File -> Open Example File`.
+    The example dataset contains three different bovine serum albumin (BSA) runs.
 		First, FeatureFinderCentroided is called since the dataset is centroided. The
 		results of the feature finding are then annotated with (existing) identification results.
 		For convenience, we provide these search results (as idXML files) with an FDR of 5% in the BSA directory.
@@ -318,22 +329,35 @@
 		@image html TOPPAS_BSA_results_3d.png 
 		@image latex TOPPAS_BSA_results_3d.png "" width=10cm 
 
+
+    @section TOPPAS_subsetneighborsearch_example Subset Neighbor Search
+
+    We will use a special FDR search strategy described by <a href="https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8489664/">Lin et al.</a>.
+    It uses a mode of @ref TOPP_DecoyDatabase which allows to create a special "neighbor" database to control the FDR.
+
+    The example pipeline is named @p FDR_NeighborSearch.toppas. Inspect it in TOPPAS via `File -> Open Example File`. A description is provided within the TOPPAS workflow.
+
+
 		@section TOPPAS_merger_example Merger and Collect nodes
 
 		The following example is actually not a useful workflow but is supposed
 		to demonstrate how merger and collector nodes can be used in a pipeline. Have a look at
-		@p merger_tutorial.toppas:
+		@p merger_tutorial.toppas. Inspect it in TOPPAS via `File -> Open Example File`.
 
 		@image html TOPPAS_example_merger.png
 		@image latex TOPPAS_example_merger.png "" width=14cm
 
-		As its name suggests, a merger merges its incoming file lists, i.e.,
-		files of all incoming edges are appended into new lists (which
-		have as many elements as the merger has incoming connections). All tools this merger has outgoing
-		connections to are called with these merged lists as input files. All incoming connections should
-		pass the same number of files (unless the corresponding preceding tool is in recycling mode).
+    In short: mergers require multiple input edges, whose data is combined bit by bit. 
+    Collectors on the other hand usually only have one input edge and combine all files from this single edge into a list in one go. The succeeding tool will be invoked only once, with a list of input files.
+
+    In detail, a @em merger merges its incoming file lists, i.e.,
+		files of all incoming edges are combined into new lists. Each file list has an many elements as the merger has incoming connections.
+    And there are as many lists as there are files(rounds) from the the preceeding(=upstream) tool. The tool downstream of the merger is invoked with 
+    these merged lists as input files. 
+    All incoming connections should pass the same number of files (unless one of the upstream nodes is in "recycling mode"), such that all merged lists have the same number of files.
+    In other words, if you have K input edges with N files each, the merger will create N output lists, with K elements each.
 
-		A collector node, on the other hand, waits for all rounds to finish before concatenating all files from all
+		A collector node, on the other hand, waits for all rounds to finish on its upstream side before concatenating all files from all
 		incoming connections into one single list. It then calls the next tool with this list of files as input.
 		This will happen exactly once during the entire pipeline run.
 

share/OpenMS/CHEMISTRY/Enzymes.xml

@@ -150,6 +150,7 @@
       <ITEM name="XTandemID" value="[K]|[X]" type="string" />
       <ITEM name="OMSSAID" value="6" type="int" />
       <ITEM name="MSGFID" value="3" type="int" />
+      <ITEM name="CometID" value="13" type="int" />
     </NODE>
     <NODE name="PepsinA">
       <ITEM name="Name" value="PepsinA" type="string" />
@@ -210,6 +211,7 @@
       <ITEM name="CTermGain" value="OH" type="string" />
       <ITEM name="PSIID" value="MS:1001915" type="string" />
       <ITEM name="XTandemID" value="[ALIV]|{P}" type="string" />
+      <ITEM name="CometID" value="14" type="int" />
     </NODE>
     <NODE name="proline endopeptidase">
       <ITEM name="Name" value="proline endopeptidase" type="string" />