Merge branch 'main' into fix-db-sync

.github/workflows/deployment.yml

@@ -38,9 +38,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-13]
+        os: [ubuntu-22.04, windows-latest, macos-13]
         include:
-          - os: ubuntu-latest
+          - os: ubuntu-22.04
             displayName: linux
             archivePortable: tar -c -C build/distribution JabRef | pigz --rsyncable > build/distribution/JabRef-portable_linux.tar.gz && rm -R build/distribution/JabRef
           - os: windows-latest

CHANGELOG.md

@@ -18,6 +18,8 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 ### Fixed
 
 - We fixed an issue where a bib file with UFF-8 charset was wrongly loaded with a different charset [forum#5369](https://discourse.jabref.org/t/jabref-5-15-opens-bib-files-with-shift-jis-encoding-instead-of-utf-8/5369/)
+- We fixed an issue where new entries were inserted in the middle of the table instead of at the end. [#12371](https://github.com/JabRef/jabref/pull/12371)
+- We fixed an issue where removing the sort from the table did not restore the original order. [#12371](https://github.com/JabRef/jabref/pull/12371)
 
 ### Removed
 

src/main/java/org/jabref/gui/maintable/MainTable.java

@@ -185,7 +185,28 @@ public MainTable(MainTableDataModel model,
         this.setItems(model.getEntriesFilteredAndSorted());
 
         // Enable sorting
-        model.getEntriesFilteredAndSorted().comparatorProperty().bind(this.comparatorProperty());
+        // Workaround for a JavaFX bug: https://bugs.openjdk.org/browse/JDK-8301761 (The sorting of the SortedList can become invalid)
+        // The default comparator of the SortedList does not consider the insertion index of entries that are equal according to the comparator.
+        // When two entries are equal based on the comparator, the entry that was inserted first should be considered smaller.
+        this.setSortPolicy(_ -> true);
+        model.getEntriesFilteredAndSorted().comparatorProperty().bind(
+                this.comparatorProperty().map(comparator -> {
+                    if (comparator == null) {
+                        return null;
+                    }
+
+                    return (entry1, entry2) -> {
+                        int result = comparator.compare(entry1, entry2);
+                        if (result != 0) {
+                            return result;
+                        }
+                        // If the entries are equal according to the comparator, compare them by their index in the database.
+                        // The comparison should ideally be based on the database index, but retrieving the index takes log(n). See {@link BibDatabase#indexOf}.
+                        // Using the entry ID is also valid since IDs are monotonically increasing.
+                        return entry1.getEntry().getId().compareTo(entry2.getEntry().getId());
+                    };
+                })
+        );
 
         // Store visual state
         new PersistenceVisualStateTable(this, mainTablePreferences.getColumnPreferences()).addListeners();

src/main/java/org/jabref/logic/importer/fileformat/PdfContentImporter.java

@@ -252,16 +252,24 @@ private boolean isFarAway(TextPosition previous, TextPosition current) {
             return Math.abs(Xgap) > XspaceThreshold && Math.abs(Ygap) > YspaceThreshold;
         }
 
-        private boolean isUnwantedText(TextPosition previousTextPosition, TextPosition textPosition) {
+        private boolean isUnwantedText(TextPosition previousTextPosition, TextPosition textPosition,
+                                       Map<Float, TextPosition> lastPositionMap, float fontSize) {
+            // This indicates that the text is at the start of the line, so it is needed.
             if (textPosition == null || previousTextPosition == null) {
                 return false;
             }
+            // We use the font size to identify titles. Blank characters don't have a font size, so we discard them.
+            // The space will be added back in the final result, but not in this method.
             if (StringUtil.isBlank(textPosition.getUnicode())) {
                 return true;
             }
-            // The title usually don't in the bottom 10% of a page.
-            return (textPosition.getPageHeight() - textPosition.getYDirAdj())
-                    < (textPosition.getPageHeight() * 0.1);
+            // Titles are generally not located in the bottom 10% of a page.
+            if ((textPosition.getPageHeight() - textPosition.getYDirAdj()) < (textPosition.getPageHeight() * 0.1)) {
+                return true;
+            }
+            // Characters in a title typically remain close together,
+            // so a distant character is unlikely to be part of the title.
+            return lastPositionMap.containsKey(fontSize) && isFarAway(lastPositionMap.get(fontSize), textPosition);
         }
 
         private Optional<String> findLargestFontText(List<TextPosition> textPositions) {
@@ -271,8 +279,7 @@ private Optional<String> findLargestFontText(List<TextPosition> textPositions) {
             for (TextPosition textPosition : textPositions) {
                 float fontSize = textPosition.getFontSizeInPt();
                 // Exclude unwanted text based on heuristics
-                if (isUnwantedText(previousTextPosition, textPosition) ||
-                    (lastPositionMap.containsKey(fontSize) && isFarAway(lastPositionMap.get(fontSize), textPosition))) {
+                if (isUnwantedText(previousTextPosition, textPosition, lastPositionMap, fontSize)) {
                     continue;
                 }
                 fontSizeTextMap.putIfAbsent(fontSize, new StringBuilder());

src/main/java/org/jabref/logic/search/query/SearchToSqlVisitor.java

@@ -207,6 +207,16 @@ public SqlQueryNode visitComparison(SearchParser.ComparisonContext ctx) {
             setFlags(searchFlags, REGULAR_EXPRESSION, true, true);
         }
 
+        // field = "" -> should find entries where the field is empty
+        // field != "" -> should find entries where the field is not empty
+        if (term.isEmpty()) {
+            if (searchFlags.contains(NEGATION)) {
+                searchFlags.remove(NEGATION);
+            } else {
+                searchFlags.add(NEGATION);
+            }
+        }
+
         return getFieldQueryNode(field.toLowerCase(Locale.ROOT), term, searchFlags);
     }
 

src/test/java/org/jabref/logic/search/query/SearchQuerySQLConversionTest.java

@@ -695,6 +695,38 @@ cte0 AS (
                             )
                         )
                         SELECT * FROM cte0 GROUP BY entryid"""
+                ),
+
+                Arguments.of(
+                        "file = \"\"",
+                        """
+                        WITH
+                        cte0 AS (
+                            SELECT main_table.entryid
+                            FROM bib_fields."tableName" AS main_table
+                            WHERE main_table.entryid NOT IN (
+                                SELECT inner_table.entryid
+                                FROM bib_fields."tableName" AS inner_table
+                                WHERE (
+                                    (inner_table.field_name = 'file') AND ((inner_table.field_value_literal ILIKE ('%%')) OR (inner_table.field_value_transformed ILIKE ('%%')))
+                                )
+                            )
+                        )
+                        SELECT * FROM cte0 GROUP BY entryid"""
+                ),
+
+                Arguments.of(
+                        "file != \"\"",
+                        """
+                        WITH
+                        cte0 AS (
+                            SELECT main_table.entryid
+                            FROM bib_fields."tableName" AS main_table
+                            WHERE (
+                                (main_table.field_name = 'file') AND ((main_table.field_value_literal ILIKE ('%%')) OR (main_table.field_value_transformed ILIKE ('%%')))
+                            )
+                        )
+                        SELECT * FROM cte0 GROUP BY entryid"""
                 )
         );
     }