From 418c0cf81540421e27169aa9c13666ba547dcd79 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Tue, 8 Oct 2024 18:29:53 +0200 Subject: [PATCH 01/44] use same name of main --- .github/workflows/macos_arm64.yaml | 152 +++++++++++++++++ .github/workflows/test_macos_arm64.yaml | 216 ------------------------ config.sh | 18 +- 3 files changed, 161 insertions(+), 225 deletions(-) create mode 100644 .github/workflows/macos_arm64.yaml delete mode 100644 .github/workflows/test_macos_arm64.yaml diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml new file mode 100644 index 00000000..245b56f7 --- /dev/null +++ b/.github/workflows/macos_arm64.yaml @@ -0,0 +1,152 @@ +--- +name: MacOS wheels ARM64 +on: [workflow_dispatch] +env: + WHEEL_SDIR: wheelhouse +jobs: + # build_pyproj: + # name: pyproj ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + # runs-on: ${{ matrix.os }} + # strategy: + # fail-fast: false + # matrix: + # os: [ "self-hosted" ] + # python: [ "3.12" ] + # platform: [ "arm64"] + # macos-target: [ "12" ] + # os-name: ["osx"] + # env: + # REPO_DIR: pyproj + # BUILD_COMMIT: 3.6.1 + # PLAT: "${{ matrix.platform }}" + # MB_PYTHON_VERSION: "${{ matrix.python }}" + # MB_ML_VER: "2014" + # BUILD_DEPENDS: "cython wheel" + # TEST_DEPENDS: "cython" + # TRAVIS_OS_NAME: ${{ matrix.os-name }} + # MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} + # steps: + # - uses: actions/checkout@v4 + # - name: Setup cmake + # run: | + # export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" + # echo $PATH + # brew install cmake + # echo "check python3" + # python3 -c "import sys; print(sys.version)" + # - name: Get code of pyproj + # env: + # REPO_DIR: pyproj + # WHEEL_SDIR: wheelhouse + # SETUPTOOLS_USE_DISTUTILS: stdlib + # BUILD_COMMIT: 3.6.1 + # run: | + # bash -x .github/workflows/build.sh + # - name: Build Wheel of pyproj + # env: + # REPO_DIR: pyproj + # WHEEL_SDIR: wheelhouse + # SETUPTOOLS_USE_DISTUTILS: stdlib + # BUILD_COMMIT: 3.6.1 + # run: | + # bash -x .github/workflows/wheel.sh + # - uses: actions/upload-artifact@v3 + # with: + # name: wheels + # path: wheelhouse + # + # build_Fiona: + # name: Fiona ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + # runs-on: ${{ matrix.os }} + # strategy: + # fail-fast: false + # matrix: + # os: [ "self-hosted" ] + # python: [ "3.9", "3.10", "3.11" ] + # platform: [ "arm64"] + # macos-target: [ "12" ] + # os-name: ["osx"] + # env: + # REPO_DIR: Fiona + # BUILD_COMMIT: 1.9.5 + # GDAL_FIONA: 3.7.3 + # PLAT: "${{ matrix.platform }}" + # MB_PYTHON_VERSION: "${{ matrix.python }}" + # MB_ML_VER: "2014" + # BUILD_DEPENDS: "oldest-supported-numpy cython wheel" + # TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" + # TRAVIS_OS_NAME: ${{ matrix.os-name }} + # MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} + # steps: + # - uses: actions/checkout@v4 + # with: + # submodules: true + # - name: Setup cmake + # run: | + # ls -lrt /opt/homebrew/bin/ + # /opt/homebrew/bin/brew install cmake + # echo "check python3" + # python3 -c "import sys; print(sys.version)" + # - name: Get code of Fiona + # env: + # REPO_DIR: Fiona + # WHEEL_SDIR: wheelhouse + # SETUPTOOLS_USE_DISTUTILS: stdlib + # BUILD_COMMIT: 1.9.5 + # run: | + # bash -x .github/workflows/build.sh + # - name: Build Wheel of Fiona + # env: + # REPO_DIR: Fiona + # WHEEL_SDIR: wheelhouse + # SETUPTOOLS_USE_DISTUTILS: stdlib + # BUILD_COMMIT: 1.9.5 + # BUILD_DEPENDS: "oldest-supported-numpy cython wheel" + # TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" + # run: | + # bash -x .github/workflows/wheel.sh + # - uses: actions/upload-artifact@v3 + # with: + # name: wheels + # path: wheelhouse + + build_GDAL: + name: GDAL ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["self-hosted"] + python: ["3.12"] + platform: ["arm64"] + macos-target: ["12"] + os-name: ["osx"] + env: + REPO_DIR: gdal + BUILD_COMMIT: v3.7.3 + WHEEL_SDIR: wheelhouse + PLAT: "${{ matrix.platform }}" + MB_PYTHON_VERSION: "${{ matrix.python }}" + MB_ML_VER: "2014" + BUILD_DEPENDS: "numpy wheel" + TEST_DEPENDS: "numpy" + TRAVIS_OS_NAME: ${{ matrix.os-name }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} + steps: + - uses: actions/checkout@v4 + - name: Setup cmake + run: | + ls -lrt /opt/homebrew/bin/ + /opt/homebrew/bin/brew install cmake + echo "check python3" + python3 -c "import sys; print(sys.version)" + - name: Get code of gdal + run: | + bash -x .github/workflows/build.sh + - name: Build Wheel of GDAL + run: | + bash -x .github/workflows/wheel.sh + - uses: actions/upload-artifact@v3 + with: + name: wheels + path: wheelhouse diff --git a/.github/workflows/test_macos_arm64.yaml b/.github/workflows/test_macos_arm64.yaml deleted file mode 100644 index b92aa1c9..00000000 --- a/.github/workflows/test_macos_arm64.yaml +++ /dev/null @@ -1,216 +0,0 @@ ---- -name: MacOS wheels and test ARM64 -on: [workflow_dispatch] -env: - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib -jobs: - build_pyproj: - name: pyproj ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ "self-hosted" ] - python: [ "3.9", "3.10", "3.11" ] - platform: [ "arm64"] - macos-target: [ "12" ] - os-name: ["osx"] - env: - REPO_DIR: pyproj - BUILD_COMMIT: 3.6.1 - PLAT: "${{ matrix.platform }}" - MB_PYTHON_VERSION: "${{ matrix.python }}" - MB_ML_VER: "2014" - BUILD_DEPENDS: "cython wheel" - TEST_DEPENDS: "cython" - TRAVIS_OS_NAME: ${{ matrix.os-name }} - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} - steps: - - uses: actions/checkout@v4 - - name: Setup cmake - run: | - export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" - echo $PATH - brew install cmake - echo "check python3" - python3 -c "import sys; print(sys.version)" - - name: Get code of pyproj - env: - REPO_DIR: pyproj - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 3.6.1 - run: | - bash -x .github/workflows/build.sh - - name: Build Wheel of pyproj - env: - REPO_DIR: pyproj - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 3.6.1 - run: | - bash -x .github/workflows/wheel.sh - - uses: actions/upload-artifact@v3 - with: - name: wheels - path: wheelhouse - - build_Fiona: - name: Fiona ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ "self-hosted" ] - python: [ "3.9", "3.10", "3.11" ] - platform: [ "arm64"] - macos-target: [ "12" ] - os-name: ["osx"] - env: - REPO_DIR: Fiona - BUILD_COMMIT: 1.9.5 - GDAL_FIONA: 3.7.3 - PLAT: "${{ matrix.platform }}" - MB_PYTHON_VERSION: "${{ matrix.python }}" - MB_ML_VER: "2014" - BUILD_DEPENDS: "oldest-supported-numpy cython wheel" - TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" - TRAVIS_OS_NAME: ${{ matrix.os-name }} - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup cmake - run: | - ls -lrt /opt/homebrew/bin/ - /opt/homebrew/bin/brew install cmake - echo "check python3" - python3 -c "import sys; print(sys.version)" - - name: Get code of Fiona - env: - REPO_DIR: Fiona - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 1.9.5 - run: | - bash -x .github/workflows/build.sh - - name: Build Wheel of Fiona - env: - REPO_DIR: Fiona - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 1.9.5 - BUILD_DEPENDS: "oldest-supported-numpy cython wheel" - TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" - run: | - bash -x .github/workflows/wheel.sh - - uses: actions/upload-artifact@v3 - with: - name: wheels - path: wheelhouse - - build_GDAL: - name: GDAL ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ "self-hosted" ] - python: [ "3.9", "3.10", "3.11" ] - platform: [ "arm64"] - macos-target: [ "12" ] - os-name: ["osx"] - env: - REPO_DIR: gdal - BUILD_COMMIT: v3.7.3 - PLAT: "${{ matrix.platform }}" - MB_PYTHON_VERSION: "${{ matrix.python }}" - MB_ML_VER: "2014" - BUILD_DEPENDS: "numpy wheel" - TEST_DEPENDS: "numpy" - TRAVIS_OS_NAME: ${{ matrix.os-name }} - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} - steps: - - uses: actions/checkout@v4 - - name: Setup cmake - run: | - ls -lrt /opt/homebrew/bin/ - /opt/homebrew/bin/brew install cmake - echo "check python3" - python3 -c "import sys; print(sys.version)" - - name: Get code of gdal - env: - REPO_DIR: gdal - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: v3.7.3 - run: | - bash -x .github/workflows/build.sh - - name: Build Wheel of GDAL - env: - REPO_DIR: gdal - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: v3.7.3 - run: | - bash -x .github/workflows/wheel.sh - - uses: actions/upload-artifact@v3 - with: - name: wheels - path: wheelhouse - - test_wheel: - needs: [build_GDAL, build_pyproj, build_Fiona] - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ "self-hosted" ] - python: [ "3.9", "3.10", "3.11" ] - platform: [ "arm64"] - macos-target: [ "12" ] - os-name: ["osx"] - env: - BRANCH_TEST: master - PYTHON_VERSION: "${{ matrix.python }}" - steps: - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - name: download artifact - uses: actions/download-artifact@v3 - id: download - with: - name: wheels - path: wheelhouse - - name: 'Echo artifact download path' - run: echo ${{steps.download.outputs.download-path}} - - name: Test wheels with python ${{ matrix.python }} - shell: bash - run: | - set -x - echo "engine installation to test wheel" - python3 -m venv $HOME/openquake - source $HOME/openquake/bin/activate - pip install -U pip wheel setuptools - pip install -r https://raw.githubusercontent.com/gem/oq-engine/ae-new_whl_fiona/requirements-py311.txt - cd ${{steps.download.outputs.download-path}} - PYCUT=`echo "${PYTHON_VERSION//.}"` - ls *${PYCUT}* - pip install --force-reinstall *${PYCUT}* - cd $HOME - git clone https://github.com/gem/oq-engine.git - cd oq-engine && git checkout ${BRANCH_TEST} - git status - pip install -e . - cd $HOME/oq-engine - oq dbserver upgrade - sleep 5 - pytest -vsx --color=yes openquake/sep/tests/test_utils.py - pytest -vsx --color=yes openquake/sep - pytest -vsx --color=yes openquake/calculators - oq engine --run https://downloads.openquake.org/jobs/risk_test.zip - pytest -vsx --color=yes openquake/hazardlib/tests/gsim/kotha_2020_test.py diff --git a/config.sh b/config.sh index d71b3b6b..f20c9b28 100644 --- a/config.sh +++ b/config.sh @@ -288,8 +288,8 @@ function pre_build { export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib export LD_RUN_PATH=$LD_RUN_PATH:/usr/local/lib fi - suppress build_nghttp2 - suppress build_openssl + build_nghttp2 + build_openssl # Remove previously installed curl. #sudo rm -rf /usr/local/lib/libcurl* if [ -n "$IS_OSX" ]; then sudo rm -rf /usr/local/lib/libcurl* ; else rm -rf /usr/local/lib/libcurl* ; fi @@ -297,14 +297,14 @@ function pre_build { suppress build_zlib build_curl build_sqlite - suppress build_tiff + build_tiff build_proj if [[ "$REPO_DIR" != "pyproj" ]]; then - suppress build_jpeg - suppress build_libpng - suppress build_jsonc - suppress build_expat - suppress build_geos + build_jpeg + build_libpng + build_jsonc + build_expat + build_geos build_gdal fi if [ -n "$IS_OSX" ]; then @@ -375,7 +375,7 @@ function build_wheel_cmd { function build_cmd { local abs_wheelhouse=$1 - python -m build -o $abs_wheelhouse + python -vv -m build -o $abs_wheelhouse } From c49a473d55bddcaea0940d52c7d4a88e183589d3 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Tue, 8 Oct 2024 19:02:42 +0200 Subject: [PATCH 02/44] change gdal compile option to test on MacOS --- config.sh | 82 +++++++++++++++++++++++++++++++---------------------- env_vars.sh | 2 +- 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/config.sh b/config.sh index f20c9b28..f3698417 100644 --- a/config.sh +++ b/config.sh @@ -1,4 +1,4 @@ -#k Custom utilities for Fiona wheels. +# Custom utilities for Fiona wheels. # # Test for OSX with [ -n "$IS_OSX" ]. @@ -100,13 +100,11 @@ function build_tiff { function build_proj { + build_sqlite CFLAGS="$CFLAGS -DPROJ_RENAME_SYMBOLS -g -O2" CXXFLAGS="$CXXFLAGS -DPROJ_RENAME_SYMBOLS -DPROJ_INTERNAL_CPP_NAMESPACE -g -O2" - env - sleep 3 if [ -e proj-stamp ]; then return; fi local cmake=cmake - build_sqlite echo "env: $PROJ_DIR and build prefix ${BUILD_PREFIX}" echo "env: $PROJ_DATA and build prefix ${BUILD_PREFIX}" # @@ -204,6 +202,9 @@ function build_curl { touch curl-stamp } +function build_pcre2 { + build_simple pcre2 $PCRE_VERSION https://github.com/PCRE2Project/pcre2/releases/download/pcre2-${PCRE_VERSION} +} function build_gdal { if [ -e gdal-stamp ]; then return; fi @@ -216,16 +217,19 @@ function build_gdal { build_sqlite build_expat build_geos + build_pcre2 + EXPAT_PREFIX=$BUILD_PREFIX CFLAGS="$CFLAGS -DPROJ_RENAME_SYMBOLS -g -O2" CXXFLAGS="$CXXFLAGS -DPROJ_RENAME_SYMBOLS -DPROJ_INTERNAL_CPP_NAMESPACE -g -O2" - EXPAT_PREFIX=$BUILD_PREFIX - if [ -n "$IS_OSX" ]; then + if [ -n "$IS_OSX" ]; then GEOS_CONFIG="-DGDAL_USE_GEOS=OFF" + PCRE2_LIB="$BUILD_PREFIX/lib/libpcre2-8.dylib" else GEOS_CONFIG="-DGDAL_USE_GEOS=ON" - fi + PCRE2_LIB="$BUILD_PREFIX/lib/libpcre2-8.so" + fi local cmake=cmake fetch_unpack http://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz @@ -239,32 +243,36 @@ function build_gdal { -DCMAKE_LIBRARY_PATH=$BUILD_PREFIX/lib \ -DCMAKE_PROGRAM_PATH=$BUILD_PREFIX/bin \ -DCMAKE_OSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ - ${GEOS_CONFIG} -# -DBUILD_SHARED_LIBS=ON \ -# -DCMAKE_BUILD_TYPE=Release \ -# -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF \ -# -DOGR_BUILD_OPTIONAL_DRIVERS=ON \ -# ${GEOS_CONFIG} \ -# -DGDAL_USE_TIFF=ON \ -# -DGDAL_USE_TIFF_INTERNAL=OFF \ -# -DGDAL_USE_GEOTIFF_INTERNAL=ON \ -# -DGDAL_ENABLE_DRIVER_GIF=ON \ -# -DGDAL_ENABLE_DRIVER_GRIB=ON \ -# -DGDAL_ENABLE_DRIVER_JPEG=ON \ -# -DGDAL_USE_ICONV=ON \ -# -DGDAL_USE_JSONC=ON \ -# -DGDAL_USE_JSONC_INTERNAL=OFF \ -# -DGDAL_USE_ZLIB=ON \ -# -DGDAL_USE_ZLIB_INTERNAL=OFF \ -# -DGDAL_ENABLE_DRIVER_PNG=ON \ -# -DGDAL_ENABLE_DRIVER_OGCAPI=OFF \ -# -DOGR_ENABLE_DRIVER_GPKG=ON \ -# -DBUILD_PYTHON_BINDINGS=OFF \ -# -DBUILD_JAVA_BINDINGS=OFF \ -# -DBUILD_CSHARP_BINDINGS=OFF \ -# -DGDAL_USE_SFCGAL=OFF \ -# -DGDAL_USE_XERCESC=OFF \ -# -DGDAL_USE_LIBXML2=OFF + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF \ + -DOGR_BUILD_OPTIONAL_DRIVERS=ON \ + ${GEOS_CONFIG} \ + -DGDAL_USE_TIFF=ON \ + -DGDAL_USE_TIFF_INTERNAL=OFF \ + -DGDAL_USE_GEOTIFF_INTERNAL=ON \ + -DGDAL_ENABLE_DRIVER_GIF=ON \ + -DGDAL_ENABLE_DRIVER_GRIB=ON \ + -DGDAL_ENABLE_DRIVER_JPEG=ON \ + -DGDAL_USE_ICONV=ON \ + -DGDAL_USE_JSONC=ON \ + -DGDAL_USE_JSONC_INTERNAL=OFF \ + -DGDAL_USE_ZLIB=ON \ + -DGDAL_USE_ZLIB_INTERNAL=OFF \ + -DGDAL_ENABLE_DRIVER_PNG=ON \ + -DGDAL_ENABLE_DRIVER_OGCAPI=OFF \ + -DOGR_ENABLE_DRIVER_GPKG=ON \ + -DBUILD_PYTHON_BINDINGS=OFF \ + -DBUILD_JAVA_BINDINGS=OFF \ + -DBUILD_CSHARP_BINDINGS=OFF \ + -DGDAL_USE_SFCGAL=OFF \ + -DGDAL_USE_XERCESC=OFF \ + -DGDAL_USE_LIBXML2=OFF \ + -DGDAL_USE_PCRE2=ON \ + -DPCRE2_INCLUDE_DIR=$BUILD_PREFIX/include \ + -DPCRE2-8_LIBRARY=$PCRE2_LIB \ + -DGDAL_USE_POSTGRESQL=OFF \ + -DGDAL_USE_ODBC=OFF \ cmake --build . -j4 (if [ -n "$IS_OSX" ]; then sudo cmake --install . ; else cmake --install .; fi)) if [ -n "$IS_OSX" ]; then @@ -369,6 +377,12 @@ function build_wheel_cmd { sleep 10 (cd $repo_dir && $cmd $wheelhouse) fi + if [ -n "$IS_OSX" ]; then + pip install delocate + delocate-listdeps --all --depending $wheelhouse/*.whl + else # manylinux + pip install auditwheel + fi repair_wheelhouse $wheelhouse } @@ -383,7 +397,7 @@ function macos_arm64_native_build_setup { # Setup native build for single arch arm_64 wheels export PLAT="arm64" # We don't want universal2 builds and only want an arm64 build - export _PYTHON_HOST_PLATFORM="macosx-11.0-arm64" + export _PYTHON_HOST_PLATFORM="macosx-13.0-arm64" export ARCHFLAGS+=" -arch arm64" $@ } diff --git a/env_vars.sh b/env_vars.sh index 6fff2b31..e17dea16 100644 --- a/env_vars.sh +++ b/env_vars.sh @@ -22,7 +22,7 @@ export PROJ_DIR=/usr/local/ export PROJ_DATA=${PROJ_DIR}share/proj export PROJ_WHEEL=true export PROJ_NETWORK=ON -export SETUPTOOLS_USE_DISTUTILS=stdlib +#export SETUPTOOLS_USE_DISTUTILS=stdlib if [[ "$REPO_DIR" == "pyproj" ]]; then export PROJ_DIR=${GITHUB_WORKSPACE}/pyproj/pyproj/proj_dir export PROJ_DATA=${PROJ_DIR}/share/proj From a7c9d7cb26738f8b51958f72c29a1861cfdc94c0 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Tue, 8 Oct 2024 19:25:03 +0200 Subject: [PATCH 03/44] remove pcre2 --- config.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/config.sh b/config.sh index f3698417..923c94f7 100644 --- a/config.sh +++ b/config.sh @@ -217,7 +217,6 @@ function build_gdal { build_sqlite build_expat build_geos - build_pcre2 EXPAT_PREFIX=$BUILD_PREFIX CFLAGS="$CFLAGS -DPROJ_RENAME_SYMBOLS -g -O2" From 406d348b561309b9aa9273e12920b8a03fe3f024 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 08:45:12 +0200 Subject: [PATCH 04/44] GDAL 3.8.x --- env_vars.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/env_vars.sh b/env_vars.sh index e17dea16..b7f37aa8 100644 --- a/env_vars.sh +++ b/env_vars.sh @@ -4,10 +4,10 @@ JPEG_VERSION=9c GEOS_VERSION=3.11.2 JSONC_VERSION=0.15 SQLITE_VERSION=3440000 -PROJ_VERSION=9.1.0 -PROJ_DATA_VER=1.15.0 -GDAL_VERSION=3.7.3 -CURL_VERSION=8.4.0 +PROJ_VERSION=9.3.1 +PROJ_DATA_VER=1.19.0 +GDAL_VERSION=3.8.4 +CURL_VERSION=8.8.0 NGHTTP2_VERSION=1.46.0 EXPAT_VERSION=2.4.9 TIFF_VERSION=4.3.0 From 7c62a1487516c4b3aeff16fe899c65dcc201a2af Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 08:47:35 +0200 Subject: [PATCH 05/44] GDAL 3.8.x also on config.sh --- .github/workflows/macos_arm64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 245b56f7..91979037 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -123,7 +123,7 @@ jobs: os-name: ["osx"] env: REPO_DIR: gdal - BUILD_COMMIT: v3.7.3 + BUILD_COMMIT: v3.8.4 WHEEL_SDIR: wheelhouse PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" From cb445e60115b61026710b4905947daf8493850d1 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 08:49:38 +0200 Subject: [PATCH 06/44] syntax error on cmake --- config.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/config.sh b/config.sh index 923c94f7..6bebadb7 100644 --- a/config.sh +++ b/config.sh @@ -267,11 +267,8 @@ function build_gdal { -DGDAL_USE_SFCGAL=OFF \ -DGDAL_USE_XERCESC=OFF \ -DGDAL_USE_LIBXML2=OFF \ - -DGDAL_USE_PCRE2=ON \ - -DPCRE2_INCLUDE_DIR=$BUILD_PREFIX/include \ - -DPCRE2-8_LIBRARY=$PCRE2_LIB \ -DGDAL_USE_POSTGRESQL=OFF \ - -DGDAL_USE_ODBC=OFF \ + -DGDAL_USE_ODBC=OFF cmake --build . -j4 (if [ -n "$IS_OSX" ]; then sudo cmake --install . ; else cmake --install .; fi)) if [ -n "$IS_OSX" ]; then From fe252106bb7126f203d4bb2daedd2d2ab985b9b5 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 09:49:31 +0200 Subject: [PATCH 07/44] test with python3.11 --- .github/workflows/macos_arm64.yaml | 4 ++-- env_vars.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 91979037..ff9855b8 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -117,9 +117,9 @@ jobs: fail-fast: false matrix: os: ["self-hosted"] - python: ["3.12"] + python: ["3.11"] platform: ["arm64"] - macos-target: ["12"] + macos-target: ["13"] os-name: ["osx"] env: REPO_DIR: gdal diff --git a/env_vars.sh b/env_vars.sh index b7f37aa8..ed3eb050 100644 --- a/env_vars.sh +++ b/env_vars.sh @@ -4,9 +4,9 @@ JPEG_VERSION=9c GEOS_VERSION=3.11.2 JSONC_VERSION=0.15 SQLITE_VERSION=3440000 -PROJ_VERSION=9.3.1 +PROJ_VERSION=9.1.0 PROJ_DATA_VER=1.19.0 -GDAL_VERSION=3.8.4 +GDAL_VERSION=3.7.3 CURL_VERSION=8.8.0 NGHTTP2_VERSION=1.46.0 EXPAT_VERSION=2.4.9 From d448671f731868662d08069ba592ad21672f88b0 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 10:15:38 +0200 Subject: [PATCH 08/44] test with build command --- .github/workflows/macos_arm64.yaml | 1 + multibuild/common_utils.sh | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index ff9855b8..34e5032b 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -146,6 +146,7 @@ jobs: - name: Build Wheel of GDAL run: | bash -x .github/workflows/wheel.sh + sleep 3600 - uses: actions/upload-artifact@v3 with: name: wheels diff --git a/multibuild/common_utils.sh b/multibuild/common_utils.sh index 8a48172c..19122fc0 100755 --- a/multibuild/common_utils.sh +++ b/multibuild/common_utils.sh @@ -327,7 +327,9 @@ function build_wheel_cmd { function pip_wheel_cmd { local abs_wheelhouse=$1 - pip wheel $(pip_opts) -w $abs_wheelhouse --no-deps . + python -vv -m build -w . -o $abs_wheelhouse + #python -m build -x -w . -o $abs_wheelhouse + #pip wheel $(pip_opts) -w $abs_wheelhouse --no-deps . } function bdist_wheel_cmd { From e13c34422b296e782c51195c3035dc53f06ffea7 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 10:59:01 +0200 Subject: [PATCH 09/44] GDAL issue: https://github.com/OSGeo/gdal/issues/822 --- multibuild/common_utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multibuild/common_utils.sh b/multibuild/common_utils.sh index 19122fc0..02a348ef 100755 --- a/multibuild/common_utils.sh +++ b/multibuild/common_utils.sh @@ -327,7 +327,7 @@ function build_wheel_cmd { function pip_wheel_cmd { local abs_wheelhouse=$1 - python -vv -m build -w . -o $abs_wheelhouse + python -vv -m build -n -w . -o $abs_wheelhouse #python -m build -x -w . -o $abs_wheelhouse #pip wheel $(pip_opts) -w $abs_wheelhouse --no-deps . } From 3b694f4749cbc7232ebe2cf9a0f40c4a4ef8327f Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 11:00:10 +0200 Subject: [PATCH 10/44] test with python3.12 --- .github/workflows/macos_arm64.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 34e5032b..27a30e39 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -117,13 +117,13 @@ jobs: fail-fast: false matrix: os: ["self-hosted"] - python: ["3.11"] + python: ["3.12"] platform: ["arm64"] macos-target: ["13"] os-name: ["osx"] env: REPO_DIR: gdal - BUILD_COMMIT: v3.8.4 + BUILD_COMMIT: v3.7.3 WHEEL_SDIR: wheelhouse PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" From f280306a5437bb6abf5c7b5933cc2f37bb022fcd Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 11:18:39 +0200 Subject: [PATCH 11/44] add test for numpy, GDAL issue: https://github.com/OSGeo/gdal/issues/8069 --- .github/workflows/macos_arm64.yaml | 4 ++-- .github/workflows/wheel.sh | 4 ++++ config.sh | 28 +++++++++++++++++++--------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 27a30e39..8d84b0e3 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -128,8 +128,8 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "numpy wheel" - TEST_DEPENDS: "numpy" + BUILD_DEPENDS: "numpy==1.26.2 setuptools wheel" + TEST_DEPENDS: "numpy==1.26.2" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: diff --git a/.github/workflows/wheel.sh b/.github/workflows/wheel.sh index 0c828f50..060823ff 100755 --- a/.github/workflows/wheel.sh +++ b/.github/workflows/wheel.sh @@ -22,4 +22,8 @@ echo "::group::Test wheel" if [[ "$REPO_DIR" == "Fiona" ]]; then install_run $PLAT fi + if [[ "$REPO_DIR" == "gdal" ]]; then + install_run $PLAT + python3 -c 'from osgeo import gdal_array' + fi echo "::endgroup::" diff --git a/config.sh b/config.sh index 6bebadb7..419688c8 100644 --- a/config.sh +++ b/config.sh @@ -320,9 +320,6 @@ function pre_build { } function run_tests { - unset GDAL_DATA - unset PROJ_LIB - unset PROJ_DATA if [ -n "$IS_OSX" ]; then export PATH=$PATH:${BUILD_PREFIX}/bin export LC_ALL=en_US.UTF-8 @@ -334,12 +331,25 @@ function run_tests { apt-get update apt-get install -y ca-certificates fi - cp -R ../Fiona/tests ./tests - python -m pip install "shapely;python_version<'3.12'" $TEST_DEPENDS - GDAL_ENABLE_DEPRECATED_DRIVER_GTM=YES python -m pytest -vv tests -k "not test_collection_zip_http and not test_mask_polygon_triangle and not test_show_versions and not test_append_or_driver_error and not [PCIDSK] and not cannot_append[FlatGeobuf]" - fio --version - fio env --formats - python ../test_fiona_issue383.py + if [[ "$REPO_DIR" == "Fiona" ]]; then + unset GDAL_DATA + unset PROJ_LIB + unset PROJ_DATA + cp -R ../Fiona/tests ./tests + python -m pip install "shapely" $TEST_DEPENDS + GDAL_ENABLE_DEPRECATED_DRIVER_GTM=YES python -m pytest -vv tests -k "not test_collection_zip_http and not test_mask_polygon_triangle and not test_show_versions and not test_append_or_driver_error and not [PCIDSK] and not cannot_append[FlatGeobuf]" + fio --version + fio env --formats + python ../test_fiona_issue383.py + fi + if [[ "$REPO_DIR" == "gdal" ]]; then + echo "Run import to test that numpy is included" + python3 -c 'from osgeo import gdal_array' + fi + + + + } From dae8bf754c7e7f8711ef6ee679f5d87bf4943ab7 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 11:48:20 +0200 Subject: [PATCH 12/44] without sleep --- .github/workflows/macos_arm64.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 8d84b0e3..3cf8f991 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -146,7 +146,6 @@ jobs: - name: Build Wheel of GDAL run: | bash -x .github/workflows/wheel.sh - sleep 3600 - uses: actions/upload-artifact@v3 with: name: wheels From 04cd7abc86348dd70529825600a700cd65713f57 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 12:11:35 +0200 Subject: [PATCH 13/44] add also Fiona and pyproj --- .github/workflows/macos_arm64.yaml | 210 ++++++++++++++--------------- 1 file changed, 105 insertions(+), 105 deletions(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 3cf8f991..1085584d 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -4,111 +4,111 @@ on: [workflow_dispatch] env: WHEEL_SDIR: wheelhouse jobs: - # build_pyproj: - # name: pyproj ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} - # runs-on: ${{ matrix.os }} - # strategy: - # fail-fast: false - # matrix: - # os: [ "self-hosted" ] - # python: [ "3.12" ] - # platform: [ "arm64"] - # macos-target: [ "12" ] - # os-name: ["osx"] - # env: - # REPO_DIR: pyproj - # BUILD_COMMIT: 3.6.1 - # PLAT: "${{ matrix.platform }}" - # MB_PYTHON_VERSION: "${{ matrix.python }}" - # MB_ML_VER: "2014" - # BUILD_DEPENDS: "cython wheel" - # TEST_DEPENDS: "cython" - # TRAVIS_OS_NAME: ${{ matrix.os-name }} - # MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} - # steps: - # - uses: actions/checkout@v4 - # - name: Setup cmake - # run: | - # export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" - # echo $PATH - # brew install cmake - # echo "check python3" - # python3 -c "import sys; print(sys.version)" - # - name: Get code of pyproj - # env: - # REPO_DIR: pyproj - # WHEEL_SDIR: wheelhouse - # SETUPTOOLS_USE_DISTUTILS: stdlib - # BUILD_COMMIT: 3.6.1 - # run: | - # bash -x .github/workflows/build.sh - # - name: Build Wheel of pyproj - # env: - # REPO_DIR: pyproj - # WHEEL_SDIR: wheelhouse - # SETUPTOOLS_USE_DISTUTILS: stdlib - # BUILD_COMMIT: 3.6.1 - # run: | - # bash -x .github/workflows/wheel.sh - # - uses: actions/upload-artifact@v3 - # with: - # name: wheels - # path: wheelhouse - # - # build_Fiona: - # name: Fiona ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} - # runs-on: ${{ matrix.os }} - # strategy: - # fail-fast: false - # matrix: - # os: [ "self-hosted" ] - # python: [ "3.9", "3.10", "3.11" ] - # platform: [ "arm64"] - # macos-target: [ "12" ] - # os-name: ["osx"] - # env: - # REPO_DIR: Fiona - # BUILD_COMMIT: 1.9.5 - # GDAL_FIONA: 3.7.3 - # PLAT: "${{ matrix.platform }}" - # MB_PYTHON_VERSION: "${{ matrix.python }}" - # MB_ML_VER: "2014" - # BUILD_DEPENDS: "oldest-supported-numpy cython wheel" - # TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" - # TRAVIS_OS_NAME: ${{ matrix.os-name }} - # MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: true - # - name: Setup cmake - # run: | - # ls -lrt /opt/homebrew/bin/ - # /opt/homebrew/bin/brew install cmake - # echo "check python3" - # python3 -c "import sys; print(sys.version)" - # - name: Get code of Fiona - # env: - # REPO_DIR: Fiona - # WHEEL_SDIR: wheelhouse - # SETUPTOOLS_USE_DISTUTILS: stdlib - # BUILD_COMMIT: 1.9.5 - # run: | - # bash -x .github/workflows/build.sh - # - name: Build Wheel of Fiona - # env: - # REPO_DIR: Fiona - # WHEEL_SDIR: wheelhouse - # SETUPTOOLS_USE_DISTUTILS: stdlib - # BUILD_COMMIT: 1.9.5 - # BUILD_DEPENDS: "oldest-supported-numpy cython wheel" - # TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" - # run: | - # bash -x .github/workflows/wheel.sh - # - uses: actions/upload-artifact@v3 - # with: - # name: wheels - # path: wheelhouse + build_pyproj: + name: pyproj ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["self-hosted"] + python: ["3.12"] + platform: ["arm64"] + macos-target: ["13"] + os-name: ["osx"] + env: + REPO_DIR: pyproj + BUILD_COMMIT: 3.6.1 + PLAT: "${{ matrix.platform }}" + MB_PYTHON_VERSION: "${{ matrix.python }}" + MB_ML_VER: "2014" + BUILD_DEPENDS: "cython wheel" + TEST_DEPENDS: "cython" + TRAVIS_OS_NAME: ${{ matrix.os-name }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} + steps: + - uses: actions/checkout@v4 + - name: Setup cmake + run: | + export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" + echo $PATH + brew install cmake + echo "check python3" + python3 -c "import sys; print(sys.version)" + - name: Get code of pyproj + env: + REPO_DIR: pyproj + WHEEL_SDIR: wheelhouse + SETUPTOOLS_USE_DISTUTILS: stdlib + BUILD_COMMIT: 3.6.1 + run: | + bash -x .github/workflows/build.sh + - name: Build Wheel of pyproj + env: + REPO_DIR: pyproj + WHEEL_SDIR: wheelhouse + SETUPTOOLS_USE_DISTUTILS: stdlib + BUILD_COMMIT: 3.6.1 + run: | + bash -x .github/workflows/wheel.sh + - uses: actions/upload-artifact@v3 + with: + name: wheels + path: wheelhouse + + build_Fiona: + name: Fiona ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["self-hosted"] + python: ["3.12"] + platform: ["arm64"] + macos-target: ["13"] + os-name: ["osx"] + env: + REPO_DIR: Fiona + BUILD_COMMIT: 1.9.5 + GDAL_FIONA: 3.7.3 + PLAT: "${{ matrix.platform }}" + MB_PYTHON_VERSION: "${{ matrix.python }}" + MB_ML_VER: "2014" + BUILD_DEPENDS: "oldest-supported-numpy cython wheel" + TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" + TRAVIS_OS_NAME: ${{ matrix.os-name }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup cmake + run: | + ls -lrt /opt/homebrew/bin/ + /opt/homebrew/bin/brew install cmake + echo "check python3" + python3 -c "import sys; print(sys.version)" + - name: Get code of Fiona + env: + REPO_DIR: Fiona + WHEEL_SDIR: wheelhouse + SETUPTOOLS_USE_DISTUTILS: stdlib + BUILD_COMMIT: 1.9.5 + run: | + bash -x .github/workflows/build.sh + - name: Build Wheel of Fiona + env: + REPO_DIR: Fiona + WHEEL_SDIR: wheelhouse + SETUPTOOLS_USE_DISTUTILS: stdlib + BUILD_COMMIT: 1.9.5 + BUILD_DEPENDS: "oldest-supported-numpy cython wheel" + TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" + run: | + bash -x .github/workflows/wheel.sh + - uses: actions/upload-artifact@v3 + with: + name: wheels + path: wheelhouse build_GDAL: name: GDAL ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} From 9aaeef7b859691a61b0f688fef778a3e5c24492a Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 12:19:57 +0200 Subject: [PATCH 14/44] add test of OQ --- .github/workflows/macos_arm64.yaml | 59 ++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 1085584d..98ac0ad1 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -150,3 +150,62 @@ jobs: with: name: wheels path: wheelhouse + + test_wheel: + needs: [build_GDAL, build_pyproj, build_Fiona] + name: test ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["self-hosted"] + python: ["3.12"] + platform: ["arm64"] + macos-target: ["13"] + os-name: ["osx"] + env: + BRANCH_TEST: ae-py312 + PYTHON_VERSION: "${{ matrix.python-version }}" + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: download artifact + uses: actions/download-artifact@v3 + id: download + with: + name: wheels + path: wheelhouse + - name: "Echo artifact download path" + run: echo ${{steps.download.outputs.download-path}} + - name: Test wheels with python ${{ matrix.python-version }} + shell: bash + run: | + set -x + echo "engine installation to test wheel" + python3 -m venv $HOME/openquake + source $HOME/openquake/bin/activate + pip install -U pip wheel setuptools + #pip install -r https://raw.githubusercontent.com/gem/oq-engine/master/requirements-py311-linux64.txt + cd ${{steps.download.outputs.download-path}} + PYCUT=`echo "${PYTHON_VERSION//.}"` + ls *${PYCUT}* + pip install --force-reinstall *${PYCUT}* + cd $HOME + git clone -b ${BRANCH_TEST} https://github.com/gem/oq-engine.git + cd oq-engine + git status + pip install -r requirements-py312-linux64.txt + pip install -e . + cd $HOME/oq-engine + oq engine --upgrade-db + sleep 5 + pip install pytest + sleep 3 + pytest -vsx --color=yes openquake/sep/tests/test_utils.py + pytest -vsx --color=yes openquake/sep + pytest -vsx --color=yes openquake/calculators + oq engine --run https://downloads.openquake.org/jobs/risk_test.zip + pytest -vsx --color=yes openquake/hazardlib/tests/gsim/kotha_2020_test.py From 41c68420969e091c0759c67e16f69222ec8310bc Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 12:22:59 +0200 Subject: [PATCH 15/44] linux only python3.12 --- .../{test_linux.yaml => linux_x86_64.yaml} | 98 ++++----- .github/workflows/test_macos_x86.yaml | 208 ------------------ .github/workflows/test_proj.yaml | 111 ---------- .github/workflows/test_windows.yaml | 118 ---------- 4 files changed, 44 insertions(+), 491 deletions(-) rename .github/workflows/{test_linux.yaml => linux_x86_64.yaml} (72%) delete mode 100644 .github/workflows/test_macos_x86.yaml delete mode 100644 .github/workflows/test_proj.yaml delete mode 100644 .github/workflows/test_windows.yaml diff --git a/.github/workflows/test_linux.yaml b/.github/workflows/linux_x86_64.yaml similarity index 72% rename from .github/workflows/test_linux.yaml rename to .github/workflows/linux_x86_64.yaml index 8099bf2e..308b94b5 100644 --- a/.github/workflows/test_linux.yaml +++ b/.github/workflows/linux_x86_64.yaml @@ -1,9 +1,17 @@ --- -name: Linux wheels and test x86_64 -on: [workflow_dispatch] +name: Linux wheels x86_64 +on: + workflow_dispatch: + inputs: + debug_enabled: + type: boolean + description: "tmate debugging enabled" + required: false + default: false + env: WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib + jobs: build_pyproj: name: pyproj ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} @@ -12,12 +20,13 @@ jobs: fail-fast: false matrix: os: ["ubuntu-22.04"] - python: ["3.10", "3.11", "3.12"] + python: ["3.12"] platform: ["x86_64"] os-name: ["jammy"] env: REPO_DIR: pyproj BUILD_COMMIT: 3.6.1 + WHEEL_SDIR: wheelhouse PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" @@ -26,29 +35,21 @@ jobs: TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: "3.10" - name: get code of pyproj - env: - REPO_DIR: pyproj - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 3.6.1 run: | echo $PATH bash -x .github/workflows/build.sh - name: Build Wheel of pyproj - env: - REPO_DIR: pyproj - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 3.6.1 run: | sudo apt install python3-virtualenv + - name: Build Wheel of pyproj + run: | bash -x .github/workflows/wheel.sh - uses: actions/upload-artifact@v3 with: @@ -62,7 +63,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-22.04"] - python: ["3.10", "3.11", "3.12"] + python: ["3.12"] platform: ["x86_64"] os-name: ["jammy"] env: @@ -77,27 +78,22 @@ jobs: TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: "3.10" + - name: Setup debugging session + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 30 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - name: get code of Fiona - env: - REPO_DIR: Fiona - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 1.9.5 run: | echo $PATH bash -x .github/workflows/build.sh - name: Build Wheel of Fiona env: - REPO_DIR: Fiona - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 1.9.5 BUILD_DEPENDS: "oldest-supported-numpy cython==0.29.26 wheel==0.34.2" TEST_DEPENDS: "oldest-supported-numpy attrs==21.4.0 pytest==6.2.5 click==8.0.3 mock boto3 packaging hypothesis==4.38.1 wheel==0.34.2 pytz" run: | @@ -116,12 +112,13 @@ jobs: fail-fast: false matrix: os: ["ubuntu-22.04"] - python: ["3.10", "3.11", "3.12"] + python: ["3.12"] platform: ["x86_64"] os-name: ["jammy"] env: REPO_DIR: gdal BUILD_COMMIT: v3.7.3 + WHEEL_SDIR: wheelhouse PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" @@ -130,26 +127,16 @@ jobs: TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: "3.10" - name: get code of GDAL - env: - REPO_DIR: gdal - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: v3.7.3 run: | bash -x .github/workflows/build.sh - name: Build Wheel of GDAL - env: - REPO_DIR: gdal - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: v3.7.3 run: | sudo apt install python3-virtualenv bash -x .github/workflows/wheel.sh @@ -165,12 +152,12 @@ jobs: matrix: python-version: ["3.12"] env: - BRANCH_TEST: master + BRANCH_TEST: ae-py312 PYTHON_VERSION: "${{ matrix.python-version }}" steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: download artifact @@ -189,19 +176,22 @@ jobs: python3 -m venv $HOME/openquake source $HOME/openquake/bin/activate pip install -U pip wheel setuptools - pip install -r https://raw.githubusercontent.com/gem/oq-engine/master/requirements-py310-linux64.txt - #cd ${{steps.download.outputs.download-path}} - #PYCUT=`echo "${PYTHON_VERSION//.}"` - #ls *${PYCUT}* - #pip install --force-reinstall *${PYCUT}* + #pip install -r https://raw.githubusercontent.com/gem/oq-engine/master/requirements-py311-linux64.txt + cd ${{steps.download.outputs.download-path}} + PYCUT=`echo "${PYTHON_VERSION//.}"` + ls *${PYCUT}* + pip install --force-reinstall *${PYCUT}* cd $HOME - git clone https://github.com/gem/oq-engine.git - cd oq-engine && git checkout ${BRANCH_TEST} + git clone -b ${BRANCH_TEST} https://github.com/gem/oq-engine.git + cd oq-engine git status + pip install -r requirements-py312-linux64.txt pip install -e . cd $HOME/oq-engine - oq dbserver upgrade + oq engine --upgrade-db sleep 5 + pip install pytest + sleep 3 pytest -vsx --color=yes openquake/sep/tests/test_utils.py pytest -vsx --color=yes openquake/sep pytest -vsx --color=yes openquake/calculators diff --git a/.github/workflows/test_macos_x86.yaml b/.github/workflows/test_macos_x86.yaml deleted file mode 100644 index 6eddc1c1..00000000 --- a/.github/workflows/test_macos_x86.yaml +++ /dev/null @@ -1,208 +0,0 @@ ---- -name: MacOS wheels and test x86_64 -on: [workflow_dispatch] -env: - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib -jobs: - build_pyproj: - name: pyproj ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ "macos-12" ] - python: [ "3.9", "3.10", "3.11" ] - platform: [ "x86_64"] - macos-target: [ "11" ] - os-name: ["osx"] - env: - REPO_DIR: pyproj - BUILD_COMMIT: 3.6.1 - PLAT: "${{ matrix.platform }}" - MB_PYTHON_VERSION: "${{ matrix.python }}" - MB_ML_VER: "2014" - BUILD_DEPENDS: "cython wheel" - TEST_DEPENDS: "cython" - TRAVIS_OS_NAME: ${{ matrix.os-name }} - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} - steps: - - uses: actions/checkout@v3 - - name: Setup cmake - run: | - brew install cmake - - name: Get code of pyproj - env: - REPO_DIR: pyproj - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 3.6.1 - run: | - bash -x .github/workflows/build.sh - - name: Build Wheel of pyproj - env: - REPO_DIR: pyproj - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 3.6.1 - run: | - bash -x .github/workflows/wheel.sh - - uses: actions/upload-artifact@v3 - with: - name: wheels - path: wheelhouse - - build_Fiona: - name: Fiona ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ "macos-12" ] - python: [ "3.9", "3.10", "3.11" ] - platform: [ "x86_64"] - macos-target: [ "11" ] - os-name: ["osx"] - env: - REPO_DIR: Fiona - BUILD_COMMIT: 1.9.5 - GDAL_FIONA: 3.7.3 - PLAT: "${{ matrix.platform }}" - MB_PYTHON_VERSION: "${{ matrix.python }}" - MB_ML_VER: "2014" - BUILD_DEPENDS: "oldest-supported-numpy cython wheel" - TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" - TRAVIS_OS_NAME: ${{ matrix.os-name }} - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup cmake - run: | - brew install cmake - echo "check python3" - python3 -c "import sys; print(sys.version)" - - name: Get code of Fiona - env: - REPO_DIR: Fiona - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 1.9.5 - run: | - bash -x .github/workflows/build.sh - - name: Build Wheel of Fiona - env: - REPO_DIR: Fiona - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 1.9.5 - BUILD_DEPENDS: "oldest-supported-numpy cython wheel" - TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" - run: | - bash -x .github/workflows/wheel.sh - - uses: actions/upload-artifact@v3 - with: - name: wheels - path: wheelhouse - - build_GDAL: - name: GDAL ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ "macos-12" ] - python: [ "3.9", "3.10", "3.11" ] - platform: [ "x86_64"] - macos-target: [ "11" ] - os-name: ["osx"] - env: - REPO_DIR: gdal - BUILD_COMMIT: v3.7.3 - PLAT: "${{ matrix.platform }}" - MB_PYTHON_VERSION: "${{ matrix.python }}" - MB_ML_VER: "2014" - BUILD_DEPENDS: "numpy wheel" - TEST_DEPENDS: "numpy" - TRAVIS_OS_NAME: ${{ matrix.os-name }} - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} - steps: - - uses: actions/checkout@v3 - - name: Setup cmake - run: | - brew install cmake - - name: Get code of gdal - env: - REPO_DIR: gdal - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: v3.7.3 - run: | - bash -x .github/workflows/build.sh - - name: Build Wheel of GDAL - env: - REPO_DIR: gdal - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: v3.7.3 - run: | - bash -x .github/workflows/wheel.sh - - uses: actions/upload-artifact@v3 - with: - name: wheels - path: wheelhouse - - test_wheel: - needs: [build_GDAL, build_pyproj, build_Fiona] - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ "macos-12" ] - python: [ "3.9", "3.10", "3.11" ] - platform: [ "x86_64"] - macos-target: [ "11" ] - os-name: ["osx"] - env: - BRANCH_TEST: master - PYTHON_VERSION: "${{ matrix.python }}" - steps: - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - name: download artifact - uses: actions/download-artifact@v3 - id: download - with: - name: wheels - path: wheelhouse - - name: 'Echo artifact download path' - run: echo ${{steps.download.outputs.download-path}} - - name: Test wheels with python ${{ matrix.python }} - shell: bash - run: | - set -x - echo "engine installation to test wheel" - python3 -m venv $HOME/openquake - source $HOME/openquake/bin/activate - pip install -U pip wheel setuptools - pip install -r https://raw.githubusercontent.com/gem/oq-engine/ae-new_whl_fiona/requirements-py311.txt - cd ${{steps.download.outputs.download-path}} - PYCUT=`echo "${PYTHON_VERSION//.}"` - ls *${PYCUT}* - pip install --force-reinstall *${PYCUT}* - cd $HOME - git clone https://github.com/gem/oq-engine.git - cd oq-engine && git checkout ${BRANCH_TEST} - git status - pip install -e . - cd $HOME/oq-engine - oq dbserver upgrade - sleep 5 - pytest -vsx --color=yes openquake/sep/tests/test_utils.py - pytest -vsx --color=yes openquake/sep - pytest -vsx --color=yes openquake/calculators - oq engine --run https://downloads.openquake.org/jobs/risk_test.zip - pytest -vsx --color=yes openquake/hazardlib/tests/gsim/kotha_2020_test.py diff --git a/.github/workflows/test_proj.yaml b/.github/workflows/test_proj.yaml deleted file mode 100644 index 2098118d..00000000 --- a/.github/workflows/test_proj.yaml +++ /dev/null @@ -1,111 +0,0 @@ ---- -name: Linux short x86_64 - -on: [workflow_dispatch] - -env: - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - -jobs: - build_pyproj: - name: pyproj ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ "ubuntu-22.04" ] - python: [ "3.9", "3.10", "3.11" ] - platform: [ "x86_64"] - os-name: ["jammy"] - env: - REPO_DIR: pyproj - BUILD_COMMIT: 3.6.1 - PLAT: "${{ matrix.platform }}" - MB_PYTHON_VERSION: "${{ matrix.python }}" - MB_ML_VER: "2014" - BUILD_DEPENDS: "cython wheel" - TEST_DEPENDS: "cython" - TRAVIS_OS_NAME: ${{ matrix.os-name }} - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: get code of pyproj - env: - REPO_DIR: pyproj - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 3.6.1 - run: | - echo $PATH - bash -x .github/workflows/build.sh - - name: Build Wheel of pyproj - env: - REPO_DIR: pyproj - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 3.6.1 - run: | - sudo apt install python3-virtualenv - bash -x .github/workflows/wheel.sh - - uses: actions/upload-artifact@v3 - with: - name: wheels - path: wheelhouse/ - - test_wheel: - needs: [build_pyproj] - runs-on: ubuntu-22.04 - strategy: - matrix: - python-version: [ "3.8", "3.9", "3.10" ] - env: - BRANCH_TEST: master - PYTHON_VERSION: "${{ matrix.python-version }}" - - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: download artifact - uses: actions/download-artifact@v3 - id: download - with: - name: wheels - path: wheelhouse/ - - name: 'Echo artifact download path' - run: echo ${{steps.download.outputs.download-path}} - - name: Test wheels with python ${{ matrix.python-version }} - shell: bash - run: | - set -x - echo "engine installation to test wheel" - python3 -m venv $HOME/openquake - source $HOME/openquake/bin/activate - pip install -U pip wheel setuptools - pip install install pytest - cd $HOME - git clone https://github.com/gem/oq-engine.git - cd oq-engine && git checkout ${BRANCH_TEST} - git status - pip install -e . - cd ${{steps.download.outputs.download-path}} - pwd - ls -lrt - PYCUT=`echo "${PYTHON_VERSION//.}"` - ls *${PYCUT}* - pip install --force-reinstall *${PYCUT}* - cd $HOME/oq-engine - oq dbserver upgrade - sleep 5 - pytest -vsx --color=yes openquake/hazardlib/tests/gsim/kotha_2020_test.py - pytest -vsx --color=yes openquake/sep/tests/test_utils.py - pytest -vsx --color=yes openquake/sep - pytest -vsx --color=yes openquake/calculators - oq engine --run https://downloads.openquake.org/jobs/risk_test.zip diff --git a/.github/workflows/test_windows.yaml b/.github/workflows/test_windows.yaml deleted file mode 100644 index b91b346f..00000000 --- a/.github/workflows/test_windows.yaml +++ /dev/null @@ -1,118 +0,0 @@ ---- -name: Windows wheels and test x86_64 -on: [workflow_dispatch] -jobs: - build_pyproj: - name: pyproj ${{ matrix.python }} ${{ matrix.os }} ${{ matrix.triplet }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ "windows-2022" ] - arch: [ "auto64" ] - python: [ "3.9", "3.10", "3.11" ] - triplet: [ "x64-windows" ] - env: - REPO_DIR: pyproj - PYPROJ_VERSION: 3.6.1 - steps: - - uses: actions/checkout@v4 - # - uses: actions/checkout@v4 - # with: - # repository: pyproj4/pyproj - # ref: '3.6.1' - - uses: actions/setup-python@v4 - with: - python-version: "${{ matrix.python }}" - - name: Install with vcpkg needed library - env: - PROJ_WHEEL: true - PROJ_NETWORK: ON - PROJ_VERSION: 9.1.0 - VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} - GDAL_VERSION: 3.7.3 - shell: bash - run: | - set -x - echo ${GITHUB_WORKSPACE} - ls -lrt ${GITHUB_WORKSPACE} - cd "$VCPKG_INSTALLATION_ROOT" - git pull > nul - ./bootstrap-vcpkg.bat -disableMetrics - ## simple usage is to copy over the file based - ## Install gdal since contains all other requirements for Fiona and proj - cp ${GITHUB_WORKSPACE}/windows/vcpkg_proj.json ${GITHUB_WORKSPACE}/windows/vcpkg.json - cat ${GITHUB_WORKSPACE}/windows/vcpkg.json - vcpkg install --feature-flags="versions,manifests" --x-manifest-root=${GITHUB_WORKSPACE}/windows --x-install-root=$VCPKG_INSTALLATION_ROOT/installed - # install ninja for cmake - #In order to use CMake + Ninja + MSVC (The Visual Studio Compiler) you need to startup a developer command prompt. - #https://stackoverflow.com/questions/76186423/how-to-setup-visual-studio-developer-command-prompt-on-github-actions - - uses: ilammy/msvc-dev-cmd@v1 - - name: Install PROJ - env: - PROJ_DIR: "C:\\proj" - PROJ_DATA: "c:\\proj\\share\\proj" - PROJ_VERSION: 9.1.0 - BUILD_PROJ: 1 - run: | - #gci env:* | sort-object name - choco install ninja cmake - $PROJ_DIR = $Env:PROJ_DIR - mkdir $PROJ_DIR - Write-Host "Installation folder of PROJ:"$PROJ_DIR - Write-Host "TEMP folder of wheels:"$Env:TEMP - cd "$Env:GITHUB_WORKSPACE" - git clone https://github.com/OSGeo/proj.git - cd proj - git checkout $Env:PROJ_VERSION - git status - mkdir build - cd build - cmake -GNinja .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_C_FLAGS="/WX" -DCMAKE_CXX_FLAGS="/WX" -DCMAKE_TOOLCHAIN_FILE="$Env:VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" -DCMAKE_INSTALL_PREFIX="$PROJ_DIR" - ninja -v - ninja install - # - $Env:Path = "C:\vcpkg\installed\x64-windows\bin;" + $Env:Path - $Env:Path = "$PROJ_DIR\bin;" + $Env:Path - # test bin on PATH - Write-Host "PATH: "$Env:Path - Write-Host "PROJ_DIR: "$Env:PROJ_DIR - Write-Host "PROJ_DATA: "$Env:PROJ_DATA - # - proj.exe - - name: build wheel - env: - PROJ_WHEEL: true - PROJ_NETWORK: ON - PROJ_VERSION: 9.1.0 - PROJ_DATA: "c:\\proj\\share\\proj" - PYPROJ_VERSION: 3.6.1 - BUILD_PROJ: 1 - run: | - # - cd "$Env:GITHUB_WORKSPACE" - git clone https://github.com/pyproj4/pyproj.git - cd pyproj - git checkout $Env:PYPROJ_VERSION - # NOT NEED: mkdir -p "$Env:GITHUB_WORKSPACE\pyproj\proj_dir\share" - Copy-Item -Path $Env:PROJ_DATA -Destination "$Env:GITHUB_WORKSPACE\pyproj\proj_dir\share\proj\" -Recurse - # - Write-Host "CHECK PROJ_DATA copy files" - dir "$Env:GITHUB_WORKSPACE\pyproj\proj_dir\share\proj" - Write-Host "Install python requirements" - python -m pip install delvewheel - python -m pip install -r .\requirements-dev.txt - python -m pip wheel . --wheel-dir="$Env:TEMP" --no-deps - cd "$Env:TEMP" - Write-Host "Folder Content" - ls - mkdir "$Env:GITHUB_WORKSPACE\wheelhouse" - $file=Get-ChildItem -Path .\ -Filter *.whl - unzip -l $file - delvewheel repair -v --wheel-dir "$Env:GITHUB_WORKSPACE\wheelhouse" $file - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - path: ./wheelhouse/*.whl - retention-days: 7 From a69e73bd71cbffd8ce67b58988acb39c2db2a4fa Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 12:43:52 +0200 Subject: [PATCH 16/44] use different way to build wheel --- multibuild/common_utils.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/multibuild/common_utils.sh b/multibuild/common_utils.sh index 02a348ef..1a97fec8 100755 --- a/multibuild/common_utils.sh +++ b/multibuild/common_utils.sh @@ -327,9 +327,10 @@ function build_wheel_cmd { function pip_wheel_cmd { local abs_wheelhouse=$1 - python -vv -m build -n -w . -o $abs_wheelhouse - #python -m build -x -w . -o $abs_wheelhouse - #pip wheel $(pip_opts) -w $abs_wheelhouse --no-deps . + if [[ "$REPO_DIR" == "gdal" ]]; then + python -vv -m build -n -w . -o $abs_wheelhouse + fi + pip wheel $(pip_opts) -w $abs_wheelhouse --no-deps . } function bdist_wheel_cmd { From 7ebfc3912806844bcc613cee459a6c6281da0622 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 13:10:34 +0200 Subject: [PATCH 17/44] remove SETUPTOOLS_USE_DISTUTILS: stdlib --- .github/workflows/macos_arm64.yaml | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 98ac0ad1..93b6386c 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -21,7 +21,7 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "cython wheel" + BUILD_DEPENDS: "cython wheel septools numpy==1.26.2" TEST_DEPENDS: "cython" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} @@ -35,19 +35,10 @@ jobs: echo "check python3" python3 -c "import sys; print(sys.version)" - name: Get code of pyproj - env: - REPO_DIR: pyproj - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 3.6.1 run: | bash -x .github/workflows/build.sh - name: Build Wheel of pyproj env: - REPO_DIR: pyproj - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 3.6.1 run: | bash -x .github/workflows/wheel.sh - uses: actions/upload-artifact@v3 @@ -73,7 +64,7 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "oldest-supported-numpy cython wheel" + BUILD_DEPENDS: "oldest-supported-numpy cython wheel septools" TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} @@ -88,21 +79,13 @@ jobs: echo "check python3" python3 -c "import sys; print(sys.version)" - name: Get code of Fiona - env: - REPO_DIR: Fiona - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib - BUILD_COMMIT: 1.9.5 run: | bash -x .github/workflows/build.sh - name: Build Wheel of Fiona env: - REPO_DIR: Fiona - WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib BUILD_COMMIT: 1.9.5 - BUILD_DEPENDS: "oldest-supported-numpy cython wheel" - TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" + BUILD_DEPENDS: "oldest-supported-numpy cython wheel numpy==1.26.2 setuptools" + TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz numpy==1.26.2" run: | bash -x .github/workflows/wheel.sh - uses: actions/upload-artifact@v3 From 4d29c060657d6b745d7cdb482c0614263600b823 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 13:13:16 +0200 Subject: [PATCH 18/44] change name for workflow --- .github/workflows/macos_x86_64.yaml | 195 ++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 .github/workflows/macos_x86_64.yaml diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml new file mode 100644 index 00000000..ccbdfe21 --- /dev/null +++ b/.github/workflows/macos_x86_64.yaml @@ -0,0 +1,195 @@ +--- +name: MacOS wheels x86_64 +on: [workflow_dispatch] +env: + WHEEL_SDIR: wheelhouse +jobs: + build_pyproj: + name: pyproj ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["macos-13"] + python: ["3.12"] + platform: ["x86_64"] + macos-target: ["13"] + os-name: ["osx"] + env: + REPO_DIR: pyproj + WHEEL_SDIR: wheelhouse + BUILD_COMMIT: 3.6.1 + PLAT: "${{ matrix.platform }}" + MB_PYTHON_VERSION: "${{ matrix.python }}" + MB_ML_VER: "2014" + BUILD_DEPENDS: "cython wheel setuptools numpy==1.26.2" + TEST_DEPENDS: "cython numpy==1.26.2" + TRAVIS_OS_NAME: ${{ matrix.os-name }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} + steps: + - uses: actions/checkout@v4 + - name: Setup cmake + run: | + export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" + echo $PATH + brew install cmake + echo "check python3" + python3 -c "import sys; print(sys.version)" + - name: Get code of pyproj + run: | + bash -x .github/workflows/build.sh + - name: Build Wheel of pyproj + run: | + bash -x .github/workflows/wheel.sh + - uses: actions/upload-artifact@v3 + with: + name: wheels + path: wheelhouse + + build_Fiona: + name: Fiona ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["macos-13"] + python: ["3.12"] + platform: ["x86_64"] + macos-target: ["13"] + os-name: ["osx"] + env: + REPO_DIR: Fiona + BUILD_COMMIT: 1.9.5 + WHEEL_SDIR: wheelhouse + GDAL_FIONA: 3.7.3 + PLAT: "${{ matrix.platform }}" + MB_PYTHON_VERSION: "${{ matrix.python }}" + MB_ML_VER: "2014" + BUILD_DEPENDS: "oldest-supported-numpy cython wheel numpy==1.26.2 setuptools" + TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz numpy==1.26.2" + TRAVIS_OS_NAME: ${{ matrix.os-name }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup cmake + run: | + ls -lrt /opt/homebrew/bin/ + /opt/homebrew/bin/brew install cmake + echo "check python3" + python3 -c "import sys; print(sys.version)" + - name: Get code of Fiona + run: | + bash -x .github/workflows/build.sh + - name: Build Wheel of Fiona + env: + BUILD_COMMIT: 1.9.5 + BUILD_DEPENDS: "oldest-supported-numpy cython wheel numpy==1.26.2 setuptools" + TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz numpy==1.26.2" + run: | + bash -x .github/workflows/wheel.sh + - uses: actions/upload-artifact@v3 + with: + name: wheels + path: wheelhouse + + build_GDAL: + name: GDAL ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["macos-13"] + python: ["3.12"] + platform: ["x86_64"] + macos-target: ["13"] + os-name: ["osx"] + env: + REPO_DIR: gdal + BUILD_COMMIT: v3.7.3 + WHEEL_SDIR: wheelhouse + PLAT: "${{ matrix.platform }}" + MB_PYTHON_VERSION: "${{ matrix.python }}" + MB_ML_VER: "2014" + BUILD_DEPENDS: "numpy==1.26.2 setuptools wheel" + TEST_DEPENDS: "numpy==1.26.2" + TRAVIS_OS_NAME: ${{ matrix.os-name }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} + steps: + - uses: actions/checkout@v4 + - name: Setup cmake + run: | + ls -lrt /opt/homebrew/bin/ + /opt/homebrew/bin/brew install cmake + echo "check python3" + python3 -c "import sys; print(sys.version)" + - name: Get code of gdal + run: | + bash -x .github/workflows/build.sh + - name: Build Wheel of GDAL + run: | + bash -x .github/workflows/wheel.sh + - uses: actions/upload-artifact@v3 + with: + name: wheels + path: wheelhouse + + test_wheel: + needs: [build_GDAL, build_pyproj, build_Fiona] + name: test ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["macos-13"] + python: ["3.12"] + platform: ["x86_64"] + macos-target: ["13"] + os-name: ["osx"] + env: + BRANCH_TEST: ae-py312 + PYTHON_VERSION: "${{ matrix.python-version }}" + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: download artifact + uses: actions/download-artifact@v3 + id: download + with: + name: wheels + path: wheelhouse + - name: "Echo artifact download path" + run: echo ${{steps.download.outputs.download-path}} + - name: Test wheels with python ${{ matrix.python-version }} + shell: bash + run: | + set -x + echo "engine installation to test wheel" + python3 -m venv $HOME/openquake + source $HOME/openquake/bin/activate + pip install -U pip wheel setuptools + #pip install -r https://raw.githubusercontent.com/gem/oq-engine/master/requirements-py311-linux64.txt + cd ${{steps.download.outputs.download-path}} + PYCUT=`echo "${PYTHON_VERSION//.}"` + ls *${PYCUT}* + pip install --force-reinstall *${PYCUT}* + cd $HOME + git clone -b ${BRANCH_TEST} https://github.com/gem/oq-engine.git + cd oq-engine + git status + pip install -r requirements-py312-linux64.txt + pip install -e . + cd $HOME/oq-engine + oq engine --upgrade-db + sleep 5 + pip install pytest + sleep 3 + pytest -vsx --color=yes openquake/sep/tests/test_utils.py + pytest -vsx --color=yes openquake/sep + pytest -vsx --color=yes openquake/calculators + oq engine --run https://downloads.openquake.org/jobs/risk_test.zip + pytest -vsx --color=yes openquake/hazardlib/tests/gsim/kotha_2020_test.py From 40de4a14fabe98510be330f38826e0c4b239fcf5 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 15:08:24 +0200 Subject: [PATCH 19/44] remove PATH on x86 --- .github/workflows/macos_x86_64.yaml | 2 -- env_vars.sh | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml index ccbdfe21..c0f50658 100644 --- a/.github/workflows/macos_x86_64.yaml +++ b/.github/workflows/macos_x86_64.yaml @@ -30,8 +30,6 @@ jobs: - uses: actions/checkout@v4 - name: Setup cmake run: | - export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:$PATH" - echo $PATH brew install cmake echo "check python3" python3 -c "import sys; print(sys.version)" diff --git a/env_vars.sh b/env_vars.sh index ed3eb050..1b8aefa5 100644 --- a/env_vars.sh +++ b/env_vars.sh @@ -7,7 +7,7 @@ SQLITE_VERSION=3440000 PROJ_VERSION=9.1.0 PROJ_DATA_VER=1.19.0 GDAL_VERSION=3.7.3 -CURL_VERSION=8.8.0 +CURL_VERSION=8.4.0 NGHTTP2_VERSION=1.46.0 EXPAT_VERSION=2.4.9 TIFF_VERSION=4.3.0 From 0bed0e826522e40d27688d9d0b95792beb0296ea Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 15:13:58 +0200 Subject: [PATCH 20/44] on x86_64 use brew without full path --- .github/workflows/macos_x86_64.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml index c0f50658..553efb9a 100644 --- a/.github/workflows/macos_x86_64.yaml +++ b/.github/workflows/macos_x86_64.yaml @@ -73,8 +73,7 @@ jobs: submodules: true - name: Setup cmake run: | - ls -lrt /opt/homebrew/bin/ - /opt/homebrew/bin/brew install cmake + brew install cmake echo "check python3" python3 -c "import sys; print(sys.version)" - name: Get code of Fiona @@ -118,8 +117,7 @@ jobs: - uses: actions/checkout@v4 - name: Setup cmake run: | - ls -lrt /opt/homebrew/bin/ - /opt/homebrew/bin/brew install cmake + brew install cmake echo "check python3" python3 -c "import sys; print(sys.version)" - name: Get code of gdal From f218950b0b199bc9e17a9132d525250a5f13233f Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 15:15:39 +0200 Subject: [PATCH 21/44] brew already installed on macos x86_64 --- .github/workflows/macos_x86_64.yaml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml index 553efb9a..680a87f5 100644 --- a/.github/workflows/macos_x86_64.yaml +++ b/.github/workflows/macos_x86_64.yaml @@ -28,11 +28,6 @@ jobs: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: - uses: actions/checkout@v4 - - name: Setup cmake - run: | - brew install cmake - echo "check python3" - python3 -c "import sys; print(sys.version)" - name: Get code of pyproj run: | bash -x .github/workflows/build.sh @@ -71,11 +66,6 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - name: Setup cmake - run: | - brew install cmake - echo "check python3" - python3 -c "import sys; print(sys.version)" - name: Get code of Fiona run: | bash -x .github/workflows/build.sh @@ -115,11 +105,6 @@ jobs: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: - uses: actions/checkout@v4 - - name: Setup cmake - run: | - brew install cmake - echo "check python3" - python3 -c "import sys; print(sys.version)" - name: Get code of gdal run: | bash -x .github/workflows/build.sh From 29d11bcd3186b500d574843d0674efafc1d636da Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 15:37:50 +0200 Subject: [PATCH 22/44] test inside the docker image and not outside --- .github/workflows/linux_x86_64.yaml | 23 +++++++++++------------ .github/workflows/wheel.sh | 1 - config.sh | 6 +----- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/linux_x86_64.yaml b/.github/workflows/linux_x86_64.yaml index 308b94b5..9b447f8a 100644 --- a/.github/workflows/linux_x86_64.yaml +++ b/.github/workflows/linux_x86_64.yaml @@ -73,8 +73,8 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "oldest-supported-numpy cython wheel" - TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" + BUILD_DEPENDS: "oldest-supported-numpy cython wheel numpy==1.26.2 setuptools" + TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz numpy==1.26.2 setuptools" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: @@ -84,18 +84,14 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.10" - - name: Setup debugging session - uses: mxschmitt/action-tmate@v3 - timeout-minutes: 30 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - name: get code of Fiona run: | echo $PATH bash -x .github/workflows/build.sh - name: Build Wheel of Fiona env: - BUILD_DEPENDS: "oldest-supported-numpy cython==0.29.26 wheel==0.34.2" - TEST_DEPENDS: "oldest-supported-numpy attrs==21.4.0 pytest==6.2.5 click==8.0.3 mock boto3 packaging hypothesis==4.38.1 wheel==0.34.2 pytz" + BUILD_DEPENDS: "oldest-supported-numpy cython==0.29.26 wheel==0.34.2 numpy==1.26.2 setuptools" + TEST_DEPENDS: "oldest-supported-numpy attrs==21.4.0 pytest==6.2.5 click==8.0.3 mock boto3 packaging hypothesis==4.38.1 wheel==0.34.2 pytz numpy==1.26.2 setuptools" run: | echo $PATH sudo apt install python3-virtualenv @@ -122,8 +118,8 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "numpy wheel" - TEST_DEPENDS: "numpy" + BUILD_DEPENDS: "wheel numpy==1.26.2 setuptools" + TEST_DEPENDS: "numpy==1.26.2 setuptools" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: @@ -140,6 +136,10 @@ jobs: run: | sudo apt install python3-virtualenv bash -x .github/workflows/wheel.sh + - name: Setup debugging session + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 90 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - uses: actions/upload-artifact@v3 with: name: wheels @@ -154,7 +154,6 @@ jobs: env: BRANCH_TEST: ae-py312 PYTHON_VERSION: "${{ matrix.python-version }}" - steps: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -183,7 +182,7 @@ jobs: pip install --force-reinstall *${PYCUT}* cd $HOME git clone -b ${BRANCH_TEST} https://github.com/gem/oq-engine.git - cd oq-engine + cd oq-engine git status pip install -r requirements-py312-linux64.txt pip install -e . diff --git a/.github/workflows/wheel.sh b/.github/workflows/wheel.sh index 060823ff..a82d4d6c 100755 --- a/.github/workflows/wheel.sh +++ b/.github/workflows/wheel.sh @@ -24,6 +24,5 @@ echo "::group::Test wheel" fi if [[ "$REPO_DIR" == "gdal" ]]; then install_run $PLAT - python3 -c 'from osgeo import gdal_array' fi echo "::endgroup::" diff --git a/config.sh b/config.sh index 419688c8..1e038f2e 100644 --- a/config.sh +++ b/config.sh @@ -268,7 +268,7 @@ function build_gdal { -DGDAL_USE_XERCESC=OFF \ -DGDAL_USE_LIBXML2=OFF \ -DGDAL_USE_POSTGRESQL=OFF \ - -DGDAL_USE_ODBC=OFF + -DGDAL_USE_ODBC=OFF cmake --build . -j4 (if [ -n "$IS_OSX" ]; then sudo cmake --install . ; else cmake --install .; fi)) if [ -n "$IS_OSX" ]; then @@ -346,10 +346,6 @@ function run_tests { echo "Run import to test that numpy is included" python3 -c 'from osgeo import gdal_array' fi - - - - } From d77811826b59c800d3a809f244618efd7f37ac1b Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 16:00:53 +0200 Subject: [PATCH 23/44] use https for ijg.org --- multibuild/library_builders.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multibuild/library_builders.sh b/multibuild/library_builders.sh index abaae0fc..a22f438f 100644 --- a/multibuild/library_builders.sh +++ b/multibuild/library_builders.sh @@ -154,7 +154,7 @@ function build_new_zlib { function build_jpeg { if [ -e jpeg-stamp ]; then return; fi - fetch_unpack http://ijg.org/files/jpegsrc.v${JPEG_VERSION}.tar.gz + fetch_unpack https://ijg.org/files/jpegsrc.v${JPEG_VERSION}.tar.gz (cd jpeg-${JPEG_VERSION} \ && ./configure --prefix=$BUILD_PREFIX \ && make -j4 \ From cba1df4aefbd9dcab0d83e8da17df9307a36608c Mon Sep 17 00:00:00 2001 From: Antonio Ettorre <56338190+vot4anto@users.noreply.github.com> Date: Wed, 9 Oct 2024 16:05:06 +0200 Subject: [PATCH 24/44] Update macos_arm64.yaml --- .github/workflows/macos_arm64.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 93b6386c..d95e99f8 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -38,7 +38,6 @@ jobs: run: | bash -x .github/workflows/build.sh - name: Build Wheel of pyproj - env: run: | bash -x .github/workflows/wheel.sh - uses: actions/upload-artifact@v3 From 4f2ecd10326b47ad59b7b5a924fa634cac7f656a Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 16:24:43 +0200 Subject: [PATCH 25/44] syntax error on setuptools --- .github/workflows/macos_arm64.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index d95e99f8..5b6488be 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -21,7 +21,7 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "cython wheel septools numpy==1.26.2" + BUILD_DEPENDS: "cython wheel setuptools numpy==1.26.2" TEST_DEPENDS: "cython" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} @@ -177,7 +177,7 @@ jobs: pip install --force-reinstall *${PYCUT}* cd $HOME git clone -b ${BRANCH_TEST} https://github.com/gem/oq-engine.git - cd oq-engine + cd oq-engine git status pip install -r requirements-py312-linux64.txt pip install -e . From e462287f243d60b8f6a6bab021cb701e80145423 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 16:43:31 +0200 Subject: [PATCH 26/44] add -v to curl --- config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.sh b/config.sh index 1e038f2e..003e9212 100644 --- a/config.sh +++ b/config.sh @@ -27,7 +27,7 @@ function fetch_unpack { ln -s $our_archive $out_archive else # Otherwise download it. - curl --insecure -L $url > $out_archive + curl -v --insecure -L $url > $out_archive fi fi # Unpack archive, refreshing contents, echoing dir and file From 1b4aec7216896a9b157d0834c59e9ca5a2272afd Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 17:00:18 +0200 Subject: [PATCH 27/44] add sleep to debug after success on wheel --- .github/workflows/macos_arm64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 5b6488be..8fd12eb9 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -163,7 +163,6 @@ jobs: - name: "Echo artifact download path" run: echo ${{steps.download.outputs.download-path}} - name: Test wheels with python ${{ matrix.python-version }} - shell: bash run: | set -x echo "engine installation to test wheel" @@ -174,6 +173,7 @@ jobs: cd ${{steps.download.outputs.download-path}} PYCUT=`echo "${PYTHON_VERSION//.}"` ls *${PYCUT}* + sleep 3600 pip install --force-reinstall *${PYCUT}* cd $HOME git clone -b ${BRANCH_TEST} https://github.com/gem/oq-engine.git From cf93916c1f6c3339cd7b03004cf2cacf81b8463a Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Wed, 9 Oct 2024 18:26:12 +0200 Subject: [PATCH 28/44] use correct name of python matrux --- .github/workflows/macos_arm64.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 8fd12eb9..79303e73 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -147,13 +147,13 @@ jobs: os-name: ["osx"] env: BRANCH_TEST: ae-py312 - PYTHON_VERSION: "${{ matrix.python-version }}" + PYTHON_VERSION: "${{ matrix.python }}" steps: - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python }} - name: download artifact uses: actions/download-artifact@v3 id: download From b65c085ba034735f3c3d6ac336af4286b8fb4e76 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre <56338190+vot4anto@users.noreply.github.com> Date: Wed, 9 Oct 2024 22:00:03 +0200 Subject: [PATCH 29/44] Update macos_x86_64.yaml with macos-12 --- .github/workflows/macos_x86_64.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml index 680a87f5..dd89966e 100644 --- a/.github/workflows/macos_x86_64.yaml +++ b/.github/workflows/macos_x86_64.yaml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-13"] + os: ["macos-12"] python: ["3.12"] platform: ["x86_64"] macos-target: ["13"] @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-13"] + os: ["macos-12"] python: ["3.12"] platform: ["x86_64"] macos-target: ["13"] @@ -87,7 +87,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-13"] + os: ["macos-12"] python: ["3.12"] platform: ["x86_64"] macos-target: ["13"] @@ -123,7 +123,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-13"] + os: ["macos-12"] python: ["3.12"] platform: ["x86_64"] macos-target: ["13"] From 74b1a21ba501cd0ec3e3f3299b6b1d4ac77ffb4a Mon Sep 17 00:00:00 2001 From: Antonio Ettorre <56338190+vot4anto@users.noreply.github.com> Date: Wed, 9 Oct 2024 22:27:21 +0200 Subject: [PATCH 30/44] Update macos_x86_64.yaml with macos-14 --- .github/workflows/macos_x86_64.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml index dd89966e..91031b69 100644 --- a/.github/workflows/macos_x86_64.yaml +++ b/.github/workflows/macos_x86_64.yaml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-12"] + os: ["macos-14"] python: ["3.12"] platform: ["x86_64"] macos-target: ["13"] @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-12"] + os: ["macos-14"] python: ["3.12"] platform: ["x86_64"] macos-target: ["13"] @@ -87,7 +87,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-12"] + os: ["macos-14"] python: ["3.12"] platform: ["x86_64"] macos-target: ["13"] @@ -123,7 +123,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-12"] + os: ["macos-14"] python: ["3.12"] platform: ["x86_64"] macos-target: ["13"] From 5dd1310a8ee9ca2a0013e53ea96d07a69cefb0c9 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre <56338190+vot4anto@users.noreply.github.com> Date: Wed, 9 Oct 2024 22:32:15 +0200 Subject: [PATCH 31/44] Update macos_x86_64.yaml macos-14 is arm64 --- .github/workflows/macos_x86_64.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml index 91031b69..680a87f5 100644 --- a/.github/workflows/macos_x86_64.yaml +++ b/.github/workflows/macos_x86_64.yaml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-14"] + os: ["macos-13"] python: ["3.12"] platform: ["x86_64"] macos-target: ["13"] @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-14"] + os: ["macos-13"] python: ["3.12"] platform: ["x86_64"] macos-target: ["13"] @@ -87,7 +87,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-14"] + os: ["macos-13"] python: ["3.12"] platform: ["x86_64"] macos-target: ["13"] @@ -123,7 +123,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["macos-14"] + os: ["macos-13"] python: ["3.12"] platform: ["x86_64"] macos-target: ["13"] From 1de41a039de068685bfabad883d8c973492a3eee Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 07:59:15 +0200 Subject: [PATCH 32/44] export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 --- .github/workflows/macos_x86_64.yaml | 2 ++ env_vars.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml index 680a87f5..7d815cc9 100644 --- a/.github/workflows/macos_x86_64.yaml +++ b/.github/workflows/macos_x86_64.yaml @@ -3,6 +3,8 @@ name: MacOS wheels x86_64 on: [workflow_dispatch] env: WHEEL_SDIR: wheelhouse + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 + jobs: build_pyproj: name: pyproj ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} diff --git a/env_vars.sh b/env_vars.sh index 1b8aefa5..cf07f0b7 100644 --- a/env_vars.sh +++ b/env_vars.sh @@ -15,6 +15,7 @@ OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source/ OPENSSL_ROOT=openssl-1.1.1w OPENSSL_HASH=cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 export MACOSX_DEPLOYMENT_TARGET=10.15 +export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 export GDAL_CONFIG=/usr/local/bin/gdal-config export PACKAGE_DATA=1 #from PROJ 9.x From 72c88d2e69b64cc6c8122db1dd54be02ff83a11c Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 08:03:09 +0200 Subject: [PATCH 33/44] do not remove xz and after reinstall it --- .github/workflows/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh index f54ee672..23de4629 100755 --- a/.github/workflows/build.sh +++ b/.github/workflows/build.sh @@ -3,7 +3,7 @@ if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then # webp, zstd, xz, libtiff cause a conflict with building webp and libtiff # curl from brew requires zstd, use system curl # if php is installed, brew tries to reinstall these after installing openblas - brew remove --ignore-dependencies webp zstd xz libtiff curl php + brew remove --ignore-dependencies webp zstd libtiff curl php fi echo "::group::Get code of project: $REPO_DIR" From 6516df14712b1e913b72406adbe95be0c8353889 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 08:16:35 +0200 Subject: [PATCH 34/44] Set the environment variable 'MACOSX_DEPLOYMENT_TARGET=13.0' to update minimum supported macOS for this wheel. --- env_vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env_vars.sh b/env_vars.sh index cf07f0b7..eec9f33e 100644 --- a/env_vars.sh +++ b/env_vars.sh @@ -14,7 +14,7 @@ TIFF_VERSION=4.3.0 OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source/ OPENSSL_ROOT=openssl-1.1.1w OPENSSL_HASH=cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 -export MACOSX_DEPLOYMENT_TARGET=10.15 +export MACOSX_DEPLOYMENT_TARGET=13.0 export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 export GDAL_CONFIG=/usr/local/bin/gdal-config export PACKAGE_DATA=1 From aa79f8eeff35d95a435660b5d30a9a48e7b8a397 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 08:52:33 +0200 Subject: [PATCH 35/44] remove sleep 3600 --- .github/workflows/macos_arm64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 79303e73..0414284c 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -173,7 +173,7 @@ jobs: cd ${{steps.download.outputs.download-path}} PYCUT=`echo "${PYTHON_VERSION//.}"` ls *${PYCUT}* - sleep 3600 + sleep 36 pip install --force-reinstall *${PYCUT}* cd $HOME git clone -b ${BRANCH_TEST} https://github.com/gem/oq-engine.git From 5dc29bc38ecde16216579ed70771872faf337e0f Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 08:55:26 +0200 Subject: [PATCH 36/44] use requirements for macos --- .github/workflows/macos_arm64.yaml | 2 +- .github/workflows/macos_x86_64.yaml | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 0414284c..a392ae3e 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -179,7 +179,7 @@ jobs: git clone -b ${BRANCH_TEST} https://github.com/gem/oq-engine.git cd oq-engine git status - pip install -r requirements-py312-linux64.txt + pip install -r requirements-py312-macos_arm64.txt pip install -e . cd $HOME/oq-engine oq engine --upgrade-db diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml index 7d815cc9..2b9c1b8f 100644 --- a/.github/workflows/macos_x86_64.yaml +++ b/.github/workflows/macos_x86_64.yaml @@ -155,16 +155,15 @@ jobs: python3 -m venv $HOME/openquake source $HOME/openquake/bin/activate pip install -U pip wheel setuptools - #pip install -r https://raw.githubusercontent.com/gem/oq-engine/master/requirements-py311-linux64.txt cd ${{steps.download.outputs.download-path}} PYCUT=`echo "${PYTHON_VERSION//.}"` ls *${PYCUT}* pip install --force-reinstall *${PYCUT}* cd $HOME git clone -b ${BRANCH_TEST} https://github.com/gem/oq-engine.git - cd oq-engine + cd oq-engine git status - pip install -r requirements-py312-linux64.txt + pip install -r requirements-py312-macos_x86_64.txt pip install -e . cd $HOME/oq-engine oq engine --upgrade-db From bfe120d3c3dc765a8511b2ab6e2f46c63584a143 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 11:11:33 +0200 Subject: [PATCH 37/44] add wheel for psutil --- .github/workflows/build.sh | 5 +++ .github/workflows/linux_x86_64.yaml | 45 ++++++++++++++++++++++- config.sh | 56 ++++++++++++++++------------- 3 files changed, 81 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh index 23de4629..1174751d 100755 --- a/.github/workflows/build.sh +++ b/.github/workflows/build.sh @@ -26,4 +26,9 @@ echo "::group::Get code of project: $REPO_DIR" # No such file or directory for GDAL 3.6.4 ls swig/python fi + if [[ "$REPO_DIR" == "psutil" ]]; then + git clone https://github.com/giampaolo/psutil.git + cd psutil + git checkout ${BUILD_COMMIT} + fi echo "::endgroup::" diff --git a/.github/workflows/linux_x86_64.yaml b/.github/workflows/linux_x86_64.yaml index 9b447f8a..e50717bc 100644 --- a/.github/workflows/linux_x86_64.yaml +++ b/.github/workflows/linux_x86_64.yaml @@ -13,6 +13,49 @@ env: WHEEL_SDIR: wheelhouse jobs: + build_psutil: + name: psutil ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-22.04"] + python: ["3.12"] + platform: ["x86_64"] + os-name: ["jammy"] + env: + REPO_DIR: psutil + BUILD_COMMIT: release-6.0.0 + WHEEL_SDIR: wheelhouse + PLAT: "${{ matrix.platform }}" + MB_PYTHON_VERSION: "${{ matrix.python }}" + MB_ML_VER: "2014" + BUILD_DEPENDS: "cython wheel setuptools" + TEST_DEPENDS: "cython" + TRAVIS_OS_NAME: ${{ matrix.os-name }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: get code of psutil + run: | + echo $PATH + bash -x .github/workflows/build.sh + - name: Build Wheel of psutil + run: | + sudo apt install python3-virtualenv + - name: Build Wheel of pyproj + run: | + bash -x .github/workflows/wheel.sh + - uses: actions/upload-artifact@v3 + with: + name: wheels + path: wheelhouse + build_pyproj: name: pyproj ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} runs-on: ${{ matrix.os }} @@ -30,7 +73,7 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "cython wheel" + BUILD_DEPENDS: "cython wheel setuptools" TEST_DEPENDS: "cython" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} diff --git a/config.sh b/config.sh index 003e9212..f342dab2 100644 --- a/config.sh +++ b/config.sh @@ -292,30 +292,32 @@ function pre_build { export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib export LD_RUN_PATH=$LD_RUN_PATH:/usr/local/lib fi - build_nghttp2 - build_openssl - # Remove previously installed curl. - #sudo rm -rf /usr/local/lib/libcurl* - if [ -n "$IS_OSX" ]; then sudo rm -rf /usr/local/lib/libcurl* ; else rm -rf /usr/local/lib/libcurl* ; fi - fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz - suppress build_zlib - build_curl - build_sqlite - build_tiff - build_proj - if [[ "$REPO_DIR" != "pyproj" ]]; then - build_jpeg - build_libpng - build_jsonc - build_expat - build_geos - build_gdal - fi - if [ -n "$IS_OSX" ]; then - export LDFLAGS="${LDFLAGS} -Wl,-rpath,${BUILD_PREFIX}/lib" - if [[ "$REPO_DIR" == "pyproj" ]]; then - export LDFLAGS="${LDFLAGS} -Wl,-rpath,${PROJ_DIR}/lib" - fi + if [[ "$REPO_DIR" != "psutil" ]]; then + build_nghttp2 + build_openssl + # Remove previously installed curl. + #sudo rm -rf /usr/local/lib/libcurl* + if [ -n "$IS_OSX" ]; then sudo rm -rf /usr/local/lib/libcurl* ; else rm -rf /usr/local/lib/libcurl* ; fi + fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz + build_zlib + build_curl + build_sqlite + build_tiff + build_proj + if [[ "$REPO_DIR" != "pyproj" ]]; then + build_jpeg + build_libpng + build_jsonc + build_expat + build_geos + build_gdal + fi + if [ -n "$IS_OSX" ]; then + export LDFLAGS="${LDFLAGS} -Wl,-rpath,${BUILD_PREFIX}/lib" + if [[ "$REPO_DIR" == "pyproj" ]]; then + export LDFLAGS="${LDFLAGS} -Wl,-rpath,${PROJ_DIR}/lib" + fi + fi fi } @@ -379,6 +381,12 @@ function build_wheel_cmd { sleep 10 (cd $repo_dir && $cmd $wheelhouse) fi + if [ "$REPO_DIR" == "psutil" ]; then + pwd + ls -lrt + sleep 10 + (cd $repo_dir && $cmd $wheelhouse) + fi if [ -n "$IS_OSX" ]; then pip install delocate delocate-listdeps --all --depending $wheelhouse/*.whl From feb280c08f49e370425ff3c9c6ae0290175acf05 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 11:16:48 +0200 Subject: [PATCH 38/44] add wheel for psutil also for MacOS --- .github/workflows/macos_arm64.yaml | 43 +++++++++++++++++++++++++++++ .github/workflows/macos_x86_64.yaml | 43 +++++++++++++++++++++++++++++ multibuild/common_utils.sh | 3 +- 3 files changed, 88 insertions(+), 1 deletion(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index a392ae3e..777b6a8b 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -4,6 +4,49 @@ on: [workflow_dispatch] env: WHEEL_SDIR: wheelhouse jobs: + build_psutil: + name: psutil ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-22.04"] + python: ["3.12"] + platform: ["x86_64"] + os-name: ["jammy"] + env: + REPO_DIR: psutil + BUILD_COMMIT: release-6.0.0 + WHEEL_SDIR: wheelhouse + PLAT: "${{ matrix.platform }}" + MB_PYTHON_VERSION: "${{ matrix.python }}" + MB_ML_VER: "2014" + BUILD_DEPENDS: "cython wheel setuptools" + TEST_DEPENDS: "cython" + TRAVIS_OS_NAME: ${{ matrix.os-name }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: get code of psutil + run: | + echo $PATH + bash -x .github/workflows/build.sh + - name: Build Wheel of psutil + run: | + sudo apt install python3-virtualenv + - name: Build Wheel of pyproj + run: | + bash -x .github/workflows/wheel.sh + - uses: actions/upload-artifact@v3 + with: + name: wheels + path: wheelhouse + build_pyproj: name: pyproj ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} runs-on: ${{ matrix.os }} diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml index 2b9c1b8f..ff8b1c0b 100644 --- a/.github/workflows/macos_x86_64.yaml +++ b/.github/workflows/macos_x86_64.yaml @@ -6,6 +6,49 @@ env: HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 jobs: + build_psutil: + name: psutil ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-22.04"] + python: ["3.12"] + platform: ["x86_64"] + os-name: ["jammy"] + env: + REPO_DIR: psutil + BUILD_COMMIT: release-6.0.0 + WHEEL_SDIR: wheelhouse + PLAT: "${{ matrix.platform }}" + MB_PYTHON_VERSION: "${{ matrix.python }}" + MB_ML_VER: "2014" + BUILD_DEPENDS: "cython wheel setuptools" + TEST_DEPENDS: "cython" + TRAVIS_OS_NAME: ${{ matrix.os-name }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: get code of psutil + run: | + echo $PATH + bash -x .github/workflows/build.sh + - name: Build Wheel of psutil + run: | + sudo apt install python3-virtualenv + - name: Build Wheel of pyproj + run: | + bash -x .github/workflows/wheel.sh + - uses: actions/upload-artifact@v3 + with: + name: wheels + path: wheelhouse + build_pyproj: name: pyproj ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.platform }} runs-on: ${{ matrix.os }} diff --git a/multibuild/common_utils.sh b/multibuild/common_utils.sh index 1a97fec8..1f851a4a 100755 --- a/multibuild/common_utils.sh +++ b/multibuild/common_utils.sh @@ -414,7 +414,8 @@ function get_os { function get_platform { # Report platform as given by uname # Use any Python that comes to hand. - python3 -c 'import platform; print(platform.uname()[4])' + #python3 -c 'import platform; print(platform.uname()[4])' + python -c 'import platform; print(platform.uname()[4])' } if [ "$(get_platform)" == x86_64 ] || \ From 7643a0045bf143463160680a0584b50da3f046cc Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 11:22:48 +0200 Subject: [PATCH 39/44] add right name on psutil job --- .github/workflows/linux_x86_64.yaml | 2 +- .github/workflows/macos_arm64.yaml | 2 +- .github/workflows/macos_x86_64.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux_x86_64.yaml b/.github/workflows/linux_x86_64.yaml index e50717bc..f532865f 100644 --- a/.github/workflows/linux_x86_64.yaml +++ b/.github/workflows/linux_x86_64.yaml @@ -48,7 +48,7 @@ jobs: - name: Build Wheel of psutil run: | sudo apt install python3-virtualenv - - name: Build Wheel of pyproj + - name: Build Wheel of psutil run: | bash -x .github/workflows/wheel.sh - uses: actions/upload-artifact@v3 diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 777b6a8b..3a2ba437 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -39,7 +39,7 @@ jobs: - name: Build Wheel of psutil run: | sudo apt install python3-virtualenv - - name: Build Wheel of pyproj + - name: Build Wheel of psutil run: | bash -x .github/workflows/wheel.sh - uses: actions/upload-artifact@v3 diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml index ff8b1c0b..96478d1c 100644 --- a/.github/workflows/macos_x86_64.yaml +++ b/.github/workflows/macos_x86_64.yaml @@ -41,7 +41,7 @@ jobs: - name: Build Wheel of psutil run: | sudo apt install python3-virtualenv - - name: Build Wheel of pyproj + - name: Build Wheel of psutil run: | bash -x .github/workflows/wheel.sh - uses: actions/upload-artifact@v3 From 29577de04127c1d305217aa4473c5e914228bd4d Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 11:27:58 +0200 Subject: [PATCH 40/44] add if for plat on osx --- env_vars.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/env_vars.sh b/env_vars.sh index eec9f33e..f14e4698 100644 --- a/env_vars.sh +++ b/env_vars.sh @@ -14,7 +14,10 @@ TIFF_VERSION=4.3.0 OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source/ OPENSSL_ROOT=openssl-1.1.1w OPENSSL_HASH=cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 -export MACOSX_DEPLOYMENT_TARGET=13.0 +export MACOSX_DEPLOYMENT_TARGET=11.0 +if [ -z "$IS_OSX" ] && [ "$PLAT" == x86_64 ]; then + export MACOSX_DEPLOYMENT_TARGET=13.0 +fi export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 export GDAL_CONFIG=/usr/local/bin/gdal-config export PACKAGE_DATA=1 From 1e5d5b89c4a1135f65bb32415dccea2b47519d75 Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 11:43:15 +0200 Subject: [PATCH 41/44] use correct plat --- .github/workflows/macos_arm64.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 3a2ba437..84348e76 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -10,10 +10,11 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-22.04"] + os: ["self-hosted"] python: ["3.12"] - platform: ["x86_64"] - os-name: ["jammy"] + platform: ["arm64"] + macos-target: ["13"] + os-name: ["osx"] env: REPO_DIR: psutil BUILD_COMMIT: release-6.0.0 From de4096915476d1e9518d5125ad64178ca816d11c Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 11:53:01 +0200 Subject: [PATCH 42/44] use correct plat also on Mac x86 --- .github/workflows/macos_arm64.yaml | 2 -- .github/workflows/macos_x86_64.yaml | 7 +++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 84348e76..60db8055 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -28,8 +28,6 @@ jobs: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: - uses: actions/checkout@v4 - with: - submodules: true - uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml index 96478d1c..ff401298 100644 --- a/.github/workflows/macos_x86_64.yaml +++ b/.github/workflows/macos_x86_64.yaml @@ -12,10 +12,11 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-22.04"] + os: ["macos-13"] python: ["3.12"] platform: ["x86_64"] - os-name: ["jammy"] + macos-target: ["13"] + os-name: ["osx"] env: REPO_DIR: psutil BUILD_COMMIT: release-6.0.0 @@ -29,8 +30,6 @@ jobs: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: - uses: actions/checkout@v4 - with: - submodules: true - uses: actions/setup-python@v5 with: python-version: "3.10" From b35b0a1f7538420f24c188365f6ca9b46810f74c Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 11:59:23 +0200 Subject: [PATCH 43/44] use numpy 1.26.4 --- .github/workflows/linux_x86_64.yaml | 12 ++++++------ .github/workflows/macos_arm64.yaml | 16 +++++----------- .github/workflows/macos_x86_64.yaml | 22 ++++++++-------------- 3 files changed, 19 insertions(+), 31 deletions(-) diff --git a/.github/workflows/linux_x86_64.yaml b/.github/workflows/linux_x86_64.yaml index f532865f..66df6e79 100644 --- a/.github/workflows/linux_x86_64.yaml +++ b/.github/workflows/linux_x86_64.yaml @@ -116,8 +116,8 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "oldest-supported-numpy cython wheel numpy==1.26.2 setuptools" - TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz numpy==1.26.2 setuptools" + BUILD_DEPENDS: "oldest-supported-numpy cython wheel numpy==1.26.4 setuptools" + TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz numpy==1.26.4 setuptools" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: @@ -133,8 +133,8 @@ jobs: bash -x .github/workflows/build.sh - name: Build Wheel of Fiona env: - BUILD_DEPENDS: "oldest-supported-numpy cython==0.29.26 wheel==0.34.2 numpy==1.26.2 setuptools" - TEST_DEPENDS: "oldest-supported-numpy attrs==21.4.0 pytest==6.2.5 click==8.0.3 mock boto3 packaging hypothesis==4.38.1 wheel==0.34.2 pytz numpy==1.26.2 setuptools" + BUILD_DEPENDS: "oldest-supported-numpy cython==0.29.26 wheel==0.34.2 numpy==1.26.4 setuptools" + TEST_DEPENDS: "oldest-supported-numpy attrs==21.4.0 pytest==6.2.5 click==8.0.3 mock boto3 packaging hypothesis==4.38.1 wheel==0.34.2 pytz numpy==1.26.4 setuptools" run: | echo $PATH sudo apt install python3-virtualenv @@ -161,8 +161,8 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "wheel numpy==1.26.2 setuptools" - TEST_DEPENDS: "numpy==1.26.2 setuptools" + BUILD_DEPENDS: "wheel numpy==1.26.4 setuptools" + TEST_DEPENDS: "numpy==1.26.4 setuptools" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: diff --git a/.github/workflows/macos_arm64.yaml b/.github/workflows/macos_arm64.yaml index 60db8055..d868e7cd 100644 --- a/.github/workflows/macos_arm64.yaml +++ b/.github/workflows/macos_arm64.yaml @@ -28,16 +28,10 @@ jobs: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - name: get code of psutil run: | echo $PATH bash -x .github/workflows/build.sh - - name: Build Wheel of psutil - run: | - sudo apt install python3-virtualenv - name: Build Wheel of psutil run: | bash -x .github/workflows/wheel.sh @@ -63,7 +57,7 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "cython wheel setuptools numpy==1.26.2" + BUILD_DEPENDS: "cython wheel setuptools numpy==1.26.4" TEST_DEPENDS: "cython" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} @@ -125,8 +119,8 @@ jobs: - name: Build Wheel of Fiona env: BUILD_COMMIT: 1.9.5 - BUILD_DEPENDS: "oldest-supported-numpy cython wheel numpy==1.26.2 setuptools" - TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz numpy==1.26.2" + BUILD_DEPENDS: "oldest-supported-numpy cython wheel numpy==1.26.4 setuptools" + TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz numpy==1.26.4" run: | bash -x .github/workflows/wheel.sh - uses: actions/upload-artifact@v3 @@ -152,8 +146,8 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "numpy==1.26.2 setuptools wheel" - TEST_DEPENDS: "numpy==1.26.2" + BUILD_DEPENDS: "numpy==1.26.4 setuptools wheel" + TEST_DEPENDS: "numpy==1.26.4" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: diff --git a/.github/workflows/macos_x86_64.yaml b/.github/workflows/macos_x86_64.yaml index ff401298..074cd1d7 100644 --- a/.github/workflows/macos_x86_64.yaml +++ b/.github/workflows/macos_x86_64.yaml @@ -30,16 +30,10 @@ jobs: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - name: get code of psutil run: | echo $PATH bash -x .github/workflows/build.sh - - name: Build Wheel of psutil - run: | - sudo apt install python3-virtualenv - name: Build Wheel of psutil run: | bash -x .github/workflows/wheel.sh @@ -66,8 +60,8 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "cython wheel setuptools numpy==1.26.2" - TEST_DEPENDS: "cython numpy==1.26.2" + BUILD_DEPENDS: "cython wheel setuptools numpy==1.26.4" + TEST_DEPENDS: "cython numpy==1.26.4" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: @@ -102,8 +96,8 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "oldest-supported-numpy cython wheel numpy==1.26.2 setuptools" - TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz numpy==1.26.2" + BUILD_DEPENDS: "oldest-supported-numpy cython wheel numpy==1.26.4 setuptools" + TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz numpy==1.26.4" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: @@ -116,8 +110,8 @@ jobs: - name: Build Wheel of Fiona env: BUILD_COMMIT: 1.9.5 - BUILD_DEPENDS: "oldest-supported-numpy cython wheel numpy==1.26.2 setuptools" - TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz numpy==1.26.2" + BUILD_DEPENDS: "oldest-supported-numpy cython wheel numpy==1.26.4 setuptools" + TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz numpy==1.26.4" run: | bash -x .github/workflows/wheel.sh - uses: actions/upload-artifact@v3 @@ -143,8 +137,8 @@ jobs: PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "numpy==1.26.2 setuptools wheel" - TEST_DEPENDS: "numpy==1.26.2" + BUILD_DEPENDS: "numpy==1.26.4 setuptools wheel" + TEST_DEPENDS: "numpy==1.26.4" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: From 7dc9769ad46cf7f812fe9f9251776ed4cf0c917e Mon Sep 17 00:00:00 2001 From: Antonio Ettorre Date: Thu, 10 Oct 2024 12:08:50 +0200 Subject: [PATCH 44/44] use || and not && --- env_vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env_vars.sh b/env_vars.sh index f14e4698..15cfb783 100644 --- a/env_vars.sh +++ b/env_vars.sh @@ -15,7 +15,7 @@ OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source/ OPENSSL_ROOT=openssl-1.1.1w OPENSSL_HASH=cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 export MACOSX_DEPLOYMENT_TARGET=11.0 -if [ -z "$IS_OSX" ] && [ "$PLAT" == x86_64 ]; then +if [ -z "$IS_OSX" ] || [ "$PLAT" == x86_64 ]; then export MACOSX_DEPLOYMENT_TARGET=13.0 fi export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1