diff --git a/autotest/gcore/gdal_stats.py b/autotest/gcore/gdal_stats.py
index 7a7dbe69a3f9..369eac85dab6 100755
--- a/autotest/gcore/gdal_stats.py
+++ b/autotest/gcore/gdal_stats.py
@@ -286,6 +286,7 @@ def stats_nodata_inf_progress_cbk(value, string, extra):
     extra[0] = value
 
 
+@pytest.mark.require_driver("HFA")
 def test_stats_nodata_inf():
 
     ds = gdal.GetDriverByName("HFA").Create(
diff --git a/autotest/gcore/hfa_rfc40.py b/autotest/gcore/hfa_rfc40.py
index 64ed11197c19..29c7613c08e5 100755
--- a/autotest/gcore/hfa_rfc40.py
+++ b/autotest/gcore/hfa_rfc40.py
@@ -34,7 +34,10 @@
 
 from osgeo import gdal
 
-pytestmark = pytest.mark.random_order(disabled=True)
+pytestmark = [
+    pytest.mark.random_order(disabled=True),
+    pytest.mark.require_driver("HFA"),
+]
 
 # All tests will be skipped if numpy is unavailable.
 np = pytest.importorskip("numpy")
diff --git a/autotest/gcore/hfa_srs.py b/autotest/gcore/hfa_srs.py
index f560996ed906..ea151b7f45ac 100755
--- a/autotest/gcore/hfa_srs.py
+++ b/autotest/gcore/hfa_srs.py
@@ -33,6 +33,8 @@
 
 from osgeo import gdal, osr
 
+pytestmark = pytest.mark.require_driver("HFA")
+
 ###############################################################################
 # Write a HFA/Imagine and read it back to check its SRS
 crs_list = [
diff --git a/autotest/gcore/hfa_write.py b/autotest/gcore/hfa_write.py
index da0ea139c08a..46c9faf0b74a 100755
--- a/autotest/gcore/hfa_write.py
+++ b/autotest/gcore/hfa_write.py
@@ -39,6 +39,8 @@
 
 from osgeo import gdal, osr
 
+pytestmark = pytest.mark.require_driver("HFA")
+
 ###############################################################################
 # test that we can write a small file with a custom layer name.
 
@@ -298,23 +300,30 @@ def test_hfa_use_rrd():
 
 
 @pytest.mark.require_driver("BMP")
-def test_hfa_update_existing_aux_overviews():
+def test_hfa_update_existing_aux_overviews(tmp_path):
+
+    tmp_filename = str(tmp_path / "hfa_update_existing_aux_overviews.bmp")
 
     with gdal.config_option("USE_RRD", "YES"):
 
-        ds = gdal.GetDriverByName("BMP").Create(
-            "tmp/hfa_update_existing_aux_overviews.bmp", 100, 100, 1
-        )
+        ds = gdal.GetDriverByName("BMP").Create(tmp_filename, 100, 100, 1)
         ds.GetRasterBand(1).Fill(255)
         ds = None
 
         # Create overviews
-        ds = gdal.Open("tmp/hfa_update_existing_aux_overviews.bmp")
-        ds.BuildOverviews("NEAR", overviewlist=[2, 4])
+        ds = gdal.Open(tmp_filename)
+        with gdaltest.disable_exceptions():
+            ret = ds.BuildOverviews("NEAR", overviewlist=[2, 4])
+        if (
+            gdal.GetLastErrorMsg()
+            == "This build does not support creating .aux overviews"
+        ):
+            pytest.skip(gdal.GetLastErrorMsg())
+        assert ret == 0
         ds = None
 
         # Save overviews checksum
-        ds = gdal.Open("tmp/hfa_update_existing_aux_overviews.bmp")
+        ds = gdal.Open(tmp_filename)
         cs_ovr0 = ds.GetRasterBand(1).GetOverview(0).Checksum()
         cs_ovr1 = ds.GetRasterBand(1).GetOverview(1).Checksum()
 
@@ -322,7 +331,7 @@ def test_hfa_update_existing_aux_overviews():
         ds.BuildOverviews("NEAR", overviewlist=[2, 4])
         ds = None
 
-        ds = gdal.Open("tmp/hfa_update_existing_aux_overviews.bmp")
+        ds = gdal.Open(tmp_filename)
         # Check overviews checksum
         new_cs_ovr0 = ds.GetRasterBand(1).GetOverview(0).Checksum()
         new_cs_ovr1 = ds.GetRasterBand(1).GetOverview(1).Checksum()
@@ -336,7 +345,7 @@ def test_hfa_update_existing_aux_overviews():
         ds.BuildOverviews("NEAR", overviewlist=[2, 4])
         ds = None
 
-        ds = gdal.Open("tmp/hfa_update_existing_aux_overviews.bmp")
+        ds = gdal.Open(tmp_filename)
         # Check overviews checksum
         new_cs_ovr0 = ds.GetRasterBand(1).GetOverview(0).Checksum()
         new_cs_ovr1 = ds.GetRasterBand(1).GetOverview(1).Checksum()
@@ -349,7 +358,7 @@ def test_hfa_update_existing_aux_overviews():
         ds.BuildOverviews("NEAR", overviewlist=[8])
         ds = None
 
-        ds = gdal.Open("tmp/hfa_update_existing_aux_overviews.bmp")
+        ds = gdal.Open(tmp_filename)
         # Check overviews checksum
         new_cs_ovr0 = ds.GetRasterBand(1).GetOverview(0).Checksum()
         new_cs_ovr1 = ds.GetRasterBand(1).GetOverview(1).Checksum()
@@ -359,7 +368,7 @@ def test_hfa_update_existing_aux_overviews():
             pytest.fail()
         ds = None
 
-        gdal.GetDriverByName("BMP").Delete("tmp/hfa_update_existing_aux_overviews.bmp")
+        gdal.GetDriverByName("BMP").Delete(tmp_filename)
 
 
 ###############################################################################
diff --git a/autotest/gcore/pam.py b/autotest/gcore/pam.py
index 0af14cd16b38..2cc3196a2604 100755
--- a/autotest/gcore/pam.py
+++ b/autotest/gcore/pam.py
@@ -175,6 +175,7 @@ def test_pam_5():
 #
 
 
+@pytest.mark.require_driver("HFA")
 def test_pam_6():
 
     ds = gdal.Open("data/f2r23.tif")
diff --git a/autotest/gcore/tiff_ovr.py b/autotest/gcore/tiff_ovr.py
index 7dd49bd0e7cf..15841622110b 100755
--- a/autotest/gcore/tiff_ovr.py
+++ b/autotest/gcore/tiff_ovr.py
@@ -287,6 +287,8 @@ def cbk(pct, _, user_data):
         callback_data=tab,
         options=["USE_RRD=YES"],
     )
+    if gdal.GetLastErrorMsg() == "This build does not support creating .aux overviews":
+        pytest.skip(gdal.GetLastErrorMsg())
     assert tab[0] == 1.0
 
     try:
diff --git a/autotest/gcore/tiff_read.py b/autotest/gcore/tiff_read.py
index 27f66fdb7914..250e067b9202 100755
--- a/autotest/gcore/tiff_read.py
+++ b/autotest/gcore/tiff_read.py
@@ -40,6 +40,8 @@
 
 from osgeo import gdal, osr
 
+pytestmark = pytest.mark.require_driver("HFA")
+
 init_list = [
     ("byte.tif", 1, 4672),
     ("uint16_sgilog.tif", 1, 4672),
diff --git a/autotest/gdrivers/hfa.py b/autotest/gdrivers/hfa.py
index bbdc7eb27324..7d18de7c6609 100755
--- a/autotest/gdrivers/hfa.py
+++ b/autotest/gdrivers/hfa.py
@@ -38,6 +38,8 @@
 
 from osgeo import gdal
 
+pytestmark = pytest.mark.require_driver("HFA")
+
 ###############################################################################
 # Verify we can read the special histogram metadata from a provided image.
 
diff --git a/autotest/gdrivers/pcidsk.py b/autotest/gdrivers/pcidsk.py
index 0a4acba081f0..52de0c695322 100755
--- a/autotest/gdrivers/pcidsk.py
+++ b/autotest/gdrivers/pcidsk.py
@@ -526,6 +526,8 @@ def test_pcidsk_external_ovr_rrd():
     with gdaltest.config_option("USE_RRD", "YES"):
         ds.BuildOverviews("NEAR", [2])
     ds = None
+    if gdal.GetLastErrorMsg() == "This build does not support creating .aux overviews":
+        pytest.skip(gdal.GetLastErrorMsg())
     assert gdal.VSIStatL("/vsimem/test.aux") is not None
     ds = gdal.Open("/vsimem/test.pix")
     assert ds.GetRasterBand(1).GetOverviewCount() == 1
diff --git a/autotest/pyscripts/test_gdalinfo_py.py b/autotest/pyscripts/test_gdalinfo_py.py
index 43bf7fe4e28e..8fab2f466084 100755
--- a/autotest/pyscripts/test_gdalinfo_py.py
+++ b/autotest/pyscripts/test_gdalinfo_py.py
@@ -141,6 +141,7 @@ def test_gdalinfo_py_5(script_path):
 # Test a dataset with overviews and RAT
 
 
+@pytest.mark.require_driver("HFA")
 def test_gdalinfo_py_6(script_path):
 
     ret = test_py_scripts.run_py_script(
diff --git a/autotest/pyscripts/test_pct.py b/autotest/pyscripts/test_pct.py
index c85ab2c76c93..c3dc1ca75402 100755
--- a/autotest/pyscripts/test_pct.py
+++ b/autotest/pyscripts/test_pct.py
@@ -248,6 +248,7 @@ def test_rgb2pct_3(script_path, tmp_path, rgb2pct2_tif):
 # Test pct2rgb with big CT (>256 entries)
 
 
+@pytest.mark.require_driver("HFA")
 def test_pct2rgb_4(script_path, tmp_path):
     gdal_array = pytest.importorskip("osgeo.gdal_array")
     try:
diff --git a/autotest/utilities/test_gdal_translate.py b/autotest/utilities/test_gdal_translate.py
index dad3024e4520..122cfc391227 100755
--- a/autotest/utilities/test_gdal_translate.py
+++ b/autotest/utilities/test_gdal_translate.py
@@ -581,6 +581,7 @@ def test_gdal_translate_20(gdal_translate_path, tmp_path):
 # in that case, they must be copied
 
 
+@pytest.mark.require_driver("HFA")
 def test_gdal_translate_21(gdal_translate_path, tmp_path):
 
     dst_img = str(tmp_path / "test_gdal_translate_21.img")
@@ -606,6 +607,7 @@ def test_gdal_translate_21(gdal_translate_path, tmp_path):
 # in that case, they must *NOT* be copied
 
 
+@pytest.mark.require_driver("HFA")
 def test_gdal_translate_22(gdal_translate_path, tmp_path):
 
     dst_img = str(tmp_path / "test_gdal_translate_22.img")
@@ -674,6 +676,7 @@ def test_gdal_translate_24(gdal_translate_path, tmp_path):
 # Test -norat
 
 
+@pytest.mark.require_driver("HFA")
 def test_gdal_translate_25(gdal_translate_path, tmp_path):
 
     dst_tif = str(tmp_path / "test_gdal_translate_25.tif")
@@ -1027,6 +1030,7 @@ def test_gdal_translate_35(gdal_translate_path, tmp_vsimem):
 # Test RAT is copied from hfa to gtiff - continuous/athematic
 
 
+@pytest.mark.require_driver("HFA")
 def test_gdal_translate_36(gdal_translate_path, tmp_path):
 
     dst_tif = str(tmp_path / "test_gdal_translate_36.tif")
@@ -1052,6 +1056,7 @@ def test_gdal_translate_36(gdal_translate_path, tmp_path):
 # Test RAT is copied from hfa to gtiff - thematic
 
 
+@pytest.mark.require_driver("HFA")
 def test_gdal_translate_37(gdal_translate_path, tmp_path):
 
     dst1_tif = str(tmp_path / "test_gdal_translate_37.tif")
diff --git a/autotest/utilities/test_gdalinfo.py b/autotest/utilities/test_gdalinfo.py
index 815be240a395..3883f73374ea 100755
--- a/autotest/utilities/test_gdalinfo.py
+++ b/autotest/utilities/test_gdalinfo.py
@@ -124,6 +124,7 @@ def test_gdalinfo_5(gdalinfo_path, tmp_path):
 # Test a dataset with overviews and RAT
 
 
+@pytest.mark.require_driver("HFA")
 def test_gdalinfo_6(gdalinfo_path):
 
     ret = gdaltest.runexternal(gdalinfo_path + " ../gdrivers/data/hfa/int.img")
@@ -541,6 +542,7 @@ def test_gdalinfo_stats(gdalinfo_path, tmp_path):
 # Test a dataset with overviews and RAT
 
 
+@pytest.mark.require_driver("HFA")
 def test_gdalinfo_33(gdalinfo_path):
 
     ret = gdaltest.runexternal(gdalinfo_path + " -json ../gdrivers/data/hfa/int.img")
diff --git a/frmts/CMakeLists.txt b/frmts/CMakeLists.txt
index a2193d4f23ef..41ef654bebd2 100644
--- a/frmts/CMakeLists.txt
+++ b/frmts/CMakeLists.txt
@@ -79,7 +79,7 @@ if (CMAKE_BUILD_TYPE MATCHES "Debug" OR GDAL_ENABLE_DRIVER_NULL)
   gdal_optional_format(null "NULL dummy driver")
 endif ()
 
-gdal_format(hfa "Erdas Imagine .img")
+gdal_optional_format(hfa "Erdas Imagine .img")
 gdal_optional_format(sdts "SDTS translator")
 gdal_optional_format(nitf "National Imagery Transmission Format")
 gdal_optional_format(gxf "GXF")
diff --git a/frmts/hfa/CMakeLists.txt b/frmts/hfa/CMakeLists.txt
index 5dc45eefb311..ec8a720e7eaa 100644
--- a/frmts/hfa/CMakeLists.txt
+++ b/frmts/hfa/CMakeLists.txt
@@ -13,11 +13,15 @@ add_gdal_driver(
           hfaopen.cpp
           hfatype.cpp
           hfa_overviews.cpp
-  BUILTIN)
+  PLUGIN_CAPABLE
+  NO_DEPS
+)
 gdal_standard_includes(gdal_HFA)
-target_include_directories(gdal_HFA PRIVATE $<TARGET_PROPERTY:PROJ::proj,INTERFACE_INCLUDE_DIRECTORIES>)
-target_compile_definitions(gdal_HFA PRIVATE $<TARGET_PROPERTY:PROJ::proj,INTERFACE_COMPILE_DEFINITIONS>)
 
-add_executable(hfatest EXCLUDE_FROM_ALL hfatest.cpp ../../ogr/ogr_proj_p.cpp)
-gdal_standard_includes(hfatest)
-target_link_libraries(hfatest PRIVATE $<TARGET_NAME:${GDAL_LIB_TARGET_NAME}> gdal_HFA PROJ::proj)
+gdal_target_link_libraries(gdal_HFA PRIVATE PROJ::proj)
+
+if (NOT GDAL_ENABLE_DRIVER_HFA_PLUGIN)
+    add_executable(hfatest EXCLUDE_FROM_ALL hfatest.cpp ../../ogr/ogr_proj_p.cpp)
+    gdal_standard_includes(hfatest)
+    target_link_libraries(hfatest PRIVATE $<TARGET_NAME:${GDAL_LIB_TARGET_NAME}> gdal_HFA PROJ::proj)
+endif()
diff --git a/gcore/CMakeLists.txt b/gcore/CMakeLists.txt
index 92892a99aa23..a61970e9e1f5 100644
--- a/gcore/CMakeLists.txt
+++ b/gcore/CMakeLists.txt
@@ -126,6 +126,10 @@ if (NOT GDAL_ENABLE_DRIVER_DERIVED)
   target_compile_definitions(gcore PRIVATE -DWITHOUT_DERIVED)
 endif ()
 
+if (NOT GDAL_ENABLE_DRIVER_HFA OR GDAL_ENABLE_DRIVER_HFA_PLUGIN)
+  target_compile_definitions(gcore PRIVATE -DNO_HFA_SUPPORT)
+endif()
+
 add_subdirectory(mdreader)
 
 # External libs now
diff --git a/gcore/gdaldefaultoverviews.cpp b/gcore/gdaldefaultoverviews.cpp
index ed73fafab2e0..795f971abe22 100644
--- a/gcore/gdaldefaultoverviews.cpp
+++ b/gcore/gdaldefaultoverviews.cpp
@@ -851,6 +851,11 @@ CPLErr GDALDefaultOverviews::BuildOverviews(
         pScaledOverviewWithoutMask);
     if (bOvrIsAux)
     {
+#ifdef NO_HFA_SUPPORT
+        CPLError(CE_Failure, CPLE_NotSupported,
+                 "This build does not support creating .aux overviews");
+        eErr = CE_Failure;
+#else
         if (nNewOverviews == 0)
         {
             /* if we call HFAAuxBuildOverviews() with nNewOverviews == 0 */
@@ -873,6 +878,7 @@ CPLErr GDALDefaultOverviews::BuildOverviews(
             if (abValidLevel[j])
                 abRequireRefresh[j] = true;
         }
+#endif
     }
 
     /* -------------------------------------------------------------------- */
diff --git a/ogr/ogr_proj_p.h b/ogr/ogr_proj_p.h
index 52870f83dd6a..aa492b0aeca0 100644
--- a/ogr/ogr_proj_p.h
+++ b/ogr/ogr_proj_p.h
@@ -39,7 +39,7 @@
 
 /*! @cond Doxygen_Suppress */
 
-PJ_CONTEXT *OSRGetProjTLSContext();
+PJ_CONTEXT CPL_DLL *OSRGetProjTLSContext();
 void OSRCleanupTLSContext();
 
 class OSRProjTLSCache