Improve performance of adding OGR dataset with many layers (fixes qgi…

…s#53525)

Prevent datasets from being released too early.

src/app/layers/qgsapplayerhandling.cpp

@@ -61,6 +61,7 @@
 #include "qgsrasterlayerelevationproperties.h"
 #include "qgsgdalutils.h"
 #include "qgstiledmeshlayer.h"
+#include "qgsogrproviderutils.h"
 
 #include <QObject>
 #include <QMessageBox>
@@ -743,6 +744,8 @@ QList<QgsMapLayer *> QgsAppLayerHandling::addSublayers( const QList<QgsProviderS
   QList< QgsMapLayer * > result;
   result.reserve( sortedLayers.size() );
 
+  QgsOgrProviderUtils::DeferDatasetClosing deferDatasetClosing;
+
   for ( const QgsProviderSublayerDetails &sublayer : std::as_const( sortedLayers ) )
   {
     QgsProviderSublayerDetails::LayerOptions options( QgsProject::instance()->transformContext() );

src/core/providers/ogr/qgsogrproviderutils.cpp

@@ -51,6 +51,8 @@ email                : nyall dot dawson at gmail dot com
 
 Q_GLOBAL_STATIC( QRecursiveMutex, sGlobalMutex )
 
+static QAtomicInt sDeferDatasetClosingCounter = 0;
+
 //! Map a dataset name to the number of opened GDAL dataset objects on it (if opened with GDALOpenWrapper, only for GPKG)
 typedef QMap< QString, int > OpenedDsCountMap;
 Q_GLOBAL_STATIC( OpenedDsCountMap, sMapCountOpenedDS )
@@ -2357,13 +2359,55 @@ QgsOgrLayerUniquePtr QgsOgrProviderUtils::getSqlLayer( QgsOgrLayer *baseLayer,
   return QgsOgrLayer::CreateForSql( ident, sql, baseLayer->ds, hSqlLayer );
 }
 
+void QgsOgrProviderUtils::incrementDeferDatasetClosingCounter()
+{
+  ++sDeferDatasetClosingCounter;
+}
+
+void QgsOgrProviderUtils::decrementDeferDatasetClosingCounter()
+{
+  if ( --sDeferDatasetClosingCounter == 0 )
+  {
+    QMutexLocker locker( sGlobalMutex() );
+    for ( auto iter = sMapSharedDS.begin(); iter != sMapSharedDS.end(); )
+    {
+      auto &datasetList = iter.value();
+      QList<QgsOgrProviderUtils::DatasetWithLayers *>::iterator itDsList = datasetList.begin();
+      while ( itDsList != datasetList.end() )
+      {
+        QgsOgrProviderUtils::DatasetWithLayers *ds = *itDsList;
+        if ( ds->refCount == 0 )
+        {
+          QgsOgrProviderUtils::GDALCloseWrapper( ds->hDS );
+          delete ds;
+          itDsList = datasetList.erase( itDsList );
+        }
+        else
+        {
+          ++itDsList;
+        }
+      }
+
+      if ( datasetList.isEmpty() )
+        iter = sMapSharedDS.erase( iter );
+      else
+        ++iter;
+    }
+  }
+}
+
+
+
 void QgsOgrProviderUtils::releaseInternal( const DatasetIdentification &ident,
     DatasetWithLayers *ds,
     bool removeFromDatasetList )
 {
 
   ds->refCount --;
-  if ( ds->refCount == 0 )
+  // Release if the ref count has dropped to 0, unless we were asked to defer
+  // closing (but defer only if the number of opened dataset is not too large)
+  if ( ds->refCount == 0 &&
+       ( sDeferDatasetClosingCounter == 0 || sMapSharedDS.size() > 100 ) )
   {
     Q_ASSERT( ds->setLayers.isEmpty() );
 

src/core/providers/ogr/qgsogrproviderutils.h

@@ -276,8 +276,30 @@ class CORE_EXPORT QgsOgrProviderUtils
      * \param ogrDriverName the OGR/GDAL driver name (e.g. "GPKG")
      */
     static bool saveConnection( const QString &path, const QString &ogrDriverName );
-};
 
+    /**
+     * Prevent immediate dataset closing when it could be closed.
+     * This is useful when opening a dataset with many layers.
+     * Must be paired with decrementDeferDatasetClosingCounter.
+     * It is recommended to use the QgsOgrProviderUtils::DeferDatasetClosing
+     * class instead to guarantee that correct pairing.
+     */
+    static void incrementDeferDatasetClosingCounter();
+
+    //! End the action started by incrementDeferDatasetClosingCounter()
+    static void decrementDeferDatasetClosingCounter();
+
+    //! Helper class for  QgsOgrProviderUtils::incrementDeferDatasetClosingCounter();
+    class DeferDatasetClosing
+    {
+      public:
+        //! Constructor: increment counter
+        DeferDatasetClosing() { QgsOgrProviderUtils::incrementDeferDatasetClosingCounter(); }
+
+        //! Destructor: decrement counter
+        ~DeferDatasetClosing() { QgsOgrProviderUtils::decrementDeferDatasetClosingCounter(); }
+    };
+};
 
 /**
  * \class QgsOgrDataset