From 9f3d0d031b23e29e3dc42dc5a24e730071e0ca15 Mon Sep 17 00:00:00 2001 From: Owen Littlejohns Date: Fri, 4 Oct 2024 22:10:38 -0400 Subject: [PATCH] DAS-1601 - Update platform for service Docker image to AMD64. --- .github/workflows/run_tests.yml | 3 + CHANGE.md | 286 +++++++++++++++++++++++++------- README.md | 2 +- bin/build-image | 7 +- bin/build-test | 4 +- bin/run-test | 6 +- docker/service.Dockerfile | 38 ++++- docker/tests.Dockerfile | 46 ++--- version.txt | 2 +- 9 files changed, 299 insertions(+), 95 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 5f601f5..9cae5e7 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -20,6 +20,9 @@ jobs: with: python-version: 3.9 + - name: Build service image + run: ./bin/build-image + - name: Build test image run: ./bin/build-test diff --git a/CHANGE.md b/CHANGE.md index 4134fb0..f25d79e 100644 --- a/CHANGE.md +++ b/CHANGE.md @@ -1,8 +1,37 @@ -[version 1.2.12] 2023-04-26 +# Changelog + +HGA follows semantic versioning. All notable changes to this project will be +documented in this file. The format is based on [Keep a +Changelog](http://keepachangelog.com/en/1.0.0/). + + +## [v1.2.13] - 2024-10-04 + +### Changed: + +* Updates base Docker image to use Miniconda, and installs GDAL via conda. +* Updates service image to be built on AMD64. + +## [v1.2.12] - 2023-04-26 + +### Changed: + * Rename function `combin_transfer` -> `perform_transforms` -* Removes unused code module `capture_operation.py` * Tidies Readme -[version 1.2.11] 2022-12-23 + +### Removed: + +* Removes unused code module `capture_operation.py` + +## [v1.2.11] - 2022-12-23 + +### Added: + +* Attempt to add more type hints and documentation strings, to enable better + understanding of the code. + +### Changed: + * Refactor `HarmonyAdapter.process_item` to move staging and STAC record generation to separate class method. * Refactor `HarmonyAdapter.process_zip` and `HarmonyAdapter.unpack_zipfile` to @@ -10,50 +39,89 @@ * Refactor `get_files_fromunzipfiles` to capture all NetCDF-4 and GeoTIFF extension options, while implementing list comprehensions for increased readability. -* Attempt to add more type hints and documentation strings, to enable better - understanding of the code. -[version 1.2.10] 2022-12-16 -* Migrate `calc_subset_envelope_window` and `box_to_shapefile` out of HarmonyAdapter. +## [v1.2.10] - 2022-12-16 + +### Added: + * Increased unit test coverage. -[version 1.2.9] 2022-12-14 +### Changed: + +* Migrate `calc_subset_envelope_window` and `box_to_shapefile` out of HarmonyAdapter. + +## [v1.2.9] - 2022-12-14 + +### Changed: + * Update base Docker image to `osgeo/gdal:ubuntu-full-3.6.0` (GDAL 3.6.0). * Update geospatial Python packages to support GDAL upgrade. -[version 1.2.8] 2022-12-13 +## [v1.2.8] - 2022-12-13 + +### Changed: + * Implement `OpenGDAL` context manager for `osgeo.gdal.Open`. -[version 1.2.7] 2022-12-10 +## [v1.2.7] - 2022-12-10 + +### Changed: + * Migration of static methods from HarmonyAdapter class to utility modules. * Implementation of Python native functions to copy, move and delete files or directories. * General linting clean-up. -[version 1.2.6] 2022-11-25 +## [v1.2.6] - 2022-11-25 + +### Changed: + * Improved detection of NetCDF-4 subdatasets. * Update harmony-service-lib-py to 1.0.22. * General code linting clean-up. -[version 1.2.5] 2022-08-05 +## [v1.2.5] - 2022-08-05 + +### Added: + +* Reimplements logging statements removed in HGA v1.2.4. + +### Changed: + * Updates `harmony-service-lib` dependency to 1.0.21, to ensure correct propagation of error messages to Harmony core. -* Reimplements logging statements removed in HGA v1.2.4. -[version 1.2.4] 2022-08-01 +## [v1.2.4] - 2022-08-01 + +### Removed: + * Removes logging that is interfering with exception handling. -[version 1.2.3] 2022-07-25 +## [v1.2.3] - 2022-07-25 + +### Changed: + * Update GDAL to version 3.4.3 + +### Removed: + * Remove unused tests and code. -[version 1.2.2] 2022-07-11 +## [v1.2.2] - 2022-07-11 + +### Changed: + * Update harmony-service-library-py to 1.0.20 -[version 1.2.1] 2022-07-11 +## [v1.2.1] - 2022-07-11 + +### Changed: + * Fix logging command so that correct exception is raised. -[version 1.2.0] 2022-07-08 +## [v1.2.0] - 2022-07-08 + +### Changed: * Modify transform.py to raise an exception when the set of requested variables is incompatible. e.g. they have different geocoordinates, geotransforms, x or y @@ -61,20 +129,33 @@ * Changes behavior of process_item to raise HGAException when a stac\_record is not created. Previously the code quietly logged a warning and succeeded. -[version 1.1.5] 2022-06-13 +## [v1.1.5] - 2022-06-13 + +### Changed: + +* Update transform.py to add a default greyscale colormap to image-type tiff. + Fixes a regression introduced in 1.1.4. +* TIFF files are no longer colored with gdaldem (as of 1.1.4), but a test has + been added to make that expectation explicit. + +## [v1.1.4] - 2022-05-20 + +### Changed: + +* Change to GDAL subsetter to use colormap URLs found in Harmony message + instead of hard coded ones. -* Update transform.py to add a default greyscale colormap to image-type tiff. Fixes a regression introduced in 1.1.4. -* TIFF files are no longer colored with gdaldem (as of 1.1.4), but a test has been added to make that expectation explicit. +## [v1.1.3] - 2022-05-09 -[version 1.1.4] 2022-05-20 +### Changed: -* Change to GDAL subsetter to use colormap URLs found in Harmony message instead of hard coded ones. +* Renames some code variables and functions, there should be no changes to the + functionality of the service. -[version 1.1.3] 2022-05-09 +## [v1.1.2] - 2022-04-21 -* Renames some code variables and functions, there should be no changes to the functionality of the service. +### Changed: -[version 1.1.2] 2022-04-21 * Updates service library dependencies. ```text @@ -83,58 +164,141 @@ geopandas 0.9.0 => 0.10.2 rasterio 1.1.5 => 1.2.10 numpy unsepcified => 1.22.3 ``` + +### Removed: + * Removes unused code from `tests` and `bin` -[version 1.1.1] 2022-03-23 -Input files with `.nc4` extensions are now recognised as NetCDF-4 files. An -exception is raised for unknown input file formats. Download error messages are -propagated more transparently to the end-user in cases of failure. +## [v1.1.1] - 2022-03-23 + +### Changed: + +* Input files with `.nc4` extensions are now recognised as NetCDF-4 files. +* An exception is raised for unknown input file formats. +* Download error messages are propagated more transparently to the end-user in + cases of failure. + +## [v1.1.0] - 2021-12-09 + +### Added: -[version 1.1.0] 2021-12-09 -New functionality added for being able to produce PNG output from NetCDF input +* New functionality added for being able to produce PNG output from NetCDF input. + +## [v1.0.17] - 2021-05-05 -[version 1.0.17] 2021-05-05 This is the final version. We will stop the development for a while. -[version 1.0.16] 2021-04-12 -This version integrates the regridding functionality. Users can set regridding parameters in the url query for the harmony api. They are outputCrs, scaleSize(xres/yres in output coordinates), scaleExtent(xmin,ymin,xmax,ymax) in output coordinates, and width/height (columnes/rows) of output file. You are not allow to use scaleSize and width/height in the same time. +## [v1.0.16] - 2021-04-12 + +### Added: + +* This version integrates the regridding functionality. Users can set regridding + parameters in the url query for the harmony API. They are outputCrs, scaleSize + (xres/yres in output coordinates), scaleExtent (xmin,ymin,xmax,ymax) in output + coordinates, and width/height (columnes/rows) of output file. You are not + allowed to use scaleSize and width/height in the same time. + +## [v1.0.15] - 2021-04-09 + +### Added: + +* This version extends the collection of datasets that gdal-subsetter can + process.The new datasets are: ALOS_PALSAR_Level2.2, ALOS_PALSAR_RTC_HIGH_RES, + and ALOS_PALSAR_RTC_LOW_RES. + +## [v1.0.14] - 2021-03-30 + +### Changed: + +* This version improves the geotiff to netcdf conversion code to make the output + netcdf file be CF compliant. For a non-rotated image, we define two 1D + coordinate variables. For a rotated image, we do not define the coordinate + variables. We could define two 2D coordinate variables for the rotated image, + but we found that the GIS software does not use the 2D coordinate variables to + decide the pixel locations. So we do not output 2D coordinate variables in + this version for sake of reducing the file size. + +## [v1.0.13] - 2021-03-16 + +### Changed: + +* This version does not define nodata if the original image does not define the + nodata. If the geotiff gile doe not define the nodata, the output netcdf file + also does not define the nodata. But the default filling_value to be used to + fill the data part of the variable before the geotiff data is written to the + variable. For geotiff/netcdf without nodata definaition inside, the GIS + software should use the mask data which has the same name as the data to + correctly display the image. + +## [v1.0.12] - 2021-03-15 + +### Changed: + +* This version is an experimental version. For the original image with byte type + data, if it does not define the nodata, the valid data range is 0-255. We map + the range of 0-255 to 0-254, and define the nodata value=255. This way the + output image get well display in the GIS software. + +## [v1.0.11] - 2021-03-02 + +### Changed: + +* This version improves the metadata in the output file. + +## [v1.0.10] - 2021-02-24 + +### Added: + +* This version marks the subset with either filling of nodata value directly to + data bands or creating the mask bands which are associated with the data bands. +* It also implements the reproject/resize with optional input of resampling method + from users. The available resampling methods are the same as those defined in + the gdal library. They are nearest, bilinear, cubic, cubicspline, lanczos, + average, rms, and mode. + +## [v1.0.9] - 2021-02-10 + +### Changed: + +* This version changes the authentication in the version 1.0.8 to new + access-token authentication. + +## [v1.0.8] - 2021-02-03 + +### Changed: + +* Modify transform.py to adopt to Harmony with STAC input. -[version 1.0.15] 2021-04-09 -This version extends the collection of datasets that gdal-subsetter can process.The new datasets are: ALOS_PALSAR_Level2.2, ALOS_PALSAR_RTC_HIGH_RES, and ALOS_PALSAR_RTC_LOW_RES. +## [v1.0.7] - 2021-01-15 -[version 1.0.14] 2021-03-30 -This version improves the geotiff to netcdf conversion code to make the output netcdf file be CF compliant. For a non-rotated image, we define two 1D coordinate variables. For a rotated image, we do not define the coordinate variables. We could define two 2D coordinate variables for the rotated image, but we found that the GIS software does not use the 2D coordinate variables to decide the pixel locations. So we do not output 2D coordinate varaibles in this version for sake of reducing the file size. +### Added: -[version 1.0.13] 2021-03-16 -This version does not define nodata if the original image does not define the nodata. If the geotiff gile doe not define the nodata, the output netcdf file also does not define the nodata. but the default filling_value to be used to fill the data part of the variable before the geotiff data is written to the variable. For geotiff/netcdf without nodata definaition inside, the GIS software should use the mask data which has the same name as the data to correctly display the image. +* Geotiff to netcdf conversion functionality has been added to the gdal-subsetter. + Users may request Harmony to output netcdf file if they provide the parameter + &format=application/x-netcdf4 in the request url. -[version 1.0.12] 2021-03-15 -This version is an experimental version. For the original image with byte type data, if it does not define the nodata, the valid data range is 0-255. We map the range of 0-255 to 0-254, and define the nodata value=255. This way the output image get well display in the GIS software. +## [v1.0.6] 2020-12-16 -[version 1.0.11] 2021-03-02 -This version imporved the metedata in the output file. +### Added: -[version 1.0.10] 2021-02-24 -This version marks the subset with either filling of nodata value directly to data bands or creating the mask bands which are associated with the data bands. It also implements the reproject/resize with optional input of resampling method from users. The available resampling methods are the same as those defined in the gdal library. They are nearest, bilinear, cubic, cubicspline, lanczos, average, rms, and mode. +* Addition of the functionality of subsetting with shapefile. The shapefile can + be point,line, and polygon feature shapefiles. It can be in three formats: + geojsion, ESRI shapefile, and KML file. -[version 1.0.9] 2021-02-10 -This version changes the authentication in the version 1.0.8 to new access-token authentication. +## [v1.0.5] - 2020-12-03 -[version 1.0.8] 2021-02-03 -modify transform.py to adopt to harmoy with STAC-catalog. +### Changed: -[version 1.0.7] 2021-01-15 -add the geotiff to netcdf conversion functuionality in the gdal-subsetter. Users may request the harmony to output netcdf file if they provide the parameter &format=application/x-netcdf4 in the request url. +* The gdal-subsetter code has been modified to work with new + harmony-service-library-py (2020/11/12), and new harmony (). -[version 1.0.6] 2020-12-16 -updated: addition of the functionality of subsetting with shapefile. The shapefile can be point,line, and polygon feature shapefiles. It can be in three format: geojsion, ESRI shapefile, and KML file. +## [v1.0.2] -[version 1.0.5] 2020-12-03 -modify the gdal-subsetter code to work with new harmony-service-library-py (2020/11/12), and new harmony () +### Changed: -[version 1.0.2] -updated: improve subset for rotated image. set values as 0 of these pixels outside the user's box. +* Improve subset for rotated image. set values as 0 of these pixels outside the + user's box. -[version 1.0.1] +## [v1.0.1] -[version 1.0.0] +## [v1.0.0] diff --git a/README.md b/README.md index f877960..fe0123e 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Creates the image `ghcr.io/nasa/harmony-gdal-adapter`. ```bash bin/build-test ``` -Creates the `nasa/harmony-gdal-adapter-test` test image. +Creates the `ghcr.io/nasa/harmony-gdal-adapter-test` test image. ### Run Tests ```bash diff --git a/bin/build-image b/bin/build-image index 9491f0c..e70d51b 100755 --- a/bin/build-image +++ b/bin/build-image @@ -1,4 +1,9 @@ #!/bin/bash +############################################################################### +# +# 2024-10-04: Updated to support amd64. +# +############################################################################### set -e @@ -9,4 +14,4 @@ find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf image="ghcr.io/nasa/harmony-gdal-adapter" tag=${1:-latest} -docker build -t ${image}:${tag} -f docker/service.Dockerfile . +docker build --platform linux/amd64 -t ${image}:${tag} -f docker/service.Dockerfile . diff --git a/bin/build-test b/bin/build-test index 4a9e9fb..b65af27 100755 --- a/bin/build-test +++ b/bin/build-test @@ -1,6 +1,6 @@ #!/bin/bash -image="nasa/harmony-gdal-adapter-test" +image="ghcr.io/nasa/harmony-gdal-adapter-test" tag=${1:-latest} @@ -13,4 +13,4 @@ fi # Build the image # -docker build -t ${image}:${tag} -f docker/tests.Dockerfile . +docker build --platform linux/amd64 -t ${image}:${tag} -f docker/tests.Dockerfile . diff --git a/bin/run-test b/bin/run-test index 9124568..8a0d6d1 100755 --- a/bin/run-test +++ b/bin/run-test @@ -1,6 +1,6 @@ #!/bin/bash ############################################################################### -# Execute the harmonyservices/harmony-gdal-adapter-test Docker image as a +# Execute the ghcr.io/nasa/harmony-gdal-adapter-test Docker image as a # container. This will run the unit test suite and check for Python linting # error. # @@ -20,7 +20,7 @@ mkdir -p coverage # Run the tests in a Docker container with mounted volumes for XML report # output and test coverage reporting -docker run --rm \ +docker run --platform linux/amd64 --rm \ -v $(pwd)/test-reports:/home/tests/reports \ -v $(pwd)/coverage:/home/tests/coverage \ - nasa/harmony-gdal-adapter-test + ghcr.io/nasa/harmony-gdal-adapter-test diff --git a/docker/service.Dockerfile b/docker/service.Dockerfile index b90a6f8..373ece4 100644 --- a/docker/service.Dockerfile +++ b/docker/service.Dockerfile @@ -1,27 +1,29 @@ ############################################################################### # -# Image for the harmonyservices/harmony-gdal-adapter service. This image +# Image for the ghcr.io/nasa/harmony-gdal-adapter service. This image # will be executed as a container, and will run a Harmony request. # The ordering of Docker commands is optimised to prevent re-running the more # time-consuming package installation when only service code is updated. # # 2022-01-27: Dockerfile created. +# 2024-10-04: Updated to Miniconda base image, and install GDAL via conda. # ############################################################################### -FROM osgeo/gdal:ubuntu-full-3.6.0 +FROM continuumio/miniconda3 WORKDIR "/home" -# Ensure Python is installed, add Pip for Python 3. -RUN ln -sf /usr/bin/python3 /usr/bin/python \ - && apt-get update \ - && apt-get install -y python3-pip +# Create conda environment +RUN conda create -y --name hga python=3.11 --channel conda-forge -q -y && conda clean -a + +# Install GDAL +RUN conda run --name hga conda install gdal=3.6.2 # Copy service requirements file into image. COPY requirements.txt . # Install Python dependencies. -RUN pip install -r requirements.txt +RUN conda run --name hga pip install --no-input --no-cache-dir -r requirements.txt # Copy service code into image COPY gdal_subsetter gdal_subsetter @@ -29,5 +31,27 @@ COPY gdal_subsetter gdal_subsetter # Copy version file into image - for use in logging COPY version.txt . +# Set conda environment for HGA, as `conda run` will not stream logging. +# Setting these environment variables is the equivalent of `conda activate`. +ENV _CE_CONDA='' \ + _CE_M='' \ + CONDA_DEFAULT_ENV=hga \ + CONDA_EXE=/opt/conda/bin/conda \ + CONDA_PREFIX=/opt/conda/envs/hga \ + CONDA_PREFIX_1=/opt/conda \ + CONDA_PROMPT_MODIFIER=(hga) \ + CONDA_PYTHON_EXE=/opt/conda/bin/python \ + CONDA_ROOT=/opt/conda \ + CONDA_SHLVL=2 \ + PATH="/opt/conda/envs/hga/bin:${PATH}" \ + SHLVL=1 + +# Set GDAL related environment variables. +ENV CPL_ZIP_ENCODING=UTF-8 \ + GDAL_DATA=/opt/conda/envs/hga/share/gdal \ + GSETTINGS_SCHEMA_DIR=/opt/conda/envs/hga/share/glib-2.0/schemas \ + GSETTINGS_SCHEMA_DIR_CONDA_BACKUP='' \ + PROJ_LIB=/opt/conda/envs/hga/share/proj + # Set entrypoint to invoke service ENTRYPOINT ["python3", "-m", "gdal_subsetter"] diff --git a/docker/tests.Dockerfile b/docker/tests.Dockerfile index e022030..c32a310 100644 --- a/docker/tests.Dockerfile +++ b/docker/tests.Dockerfile @@ -5,31 +5,39 @@ # a `pylint` check on the code contained in the `gdal_subsetter` directory. # # 2022-01-27: Dockerfile created. +# 2024-10-04: Updated to the service image as a base. # ############################################################################### -FROM osgeo/gdal:ubuntu-full-3.6.0 +FROM ghcr.io/nasa/harmony-gdal-adapter -WORKDIR "/home" - -# Ensure Python is installed, add Pip for Python 3. -RUN ln -sf /usr/bin/python3 /usr/bin/python \ - && apt-get update \ - && apt-get install -y python3-pip - -# Copy both service and testing requirements files into image. -COPY requirements*.txt /home/ - -# Ensure Python is installed, add Pip for Python3, install dependencies. -RUN pip install -r requirements.txt -r requirements_dev.txt - -# Copy service code into image -COPY gdal_subsetter gdal_subsetter - -# Copy version file into image - for use in logging -COPY version.txt . +# Install test requirements in hga conda environment. +COPY requirements_dev.txt /home/ +RUN conda run --name hga pip install -r requirements_dev.txt # Copy test directory into image COPY tests tests +# Set conda environment to maskfill, as `conda run` will not stream logging. +# Setting these environment variables is the equivalent of `conda activate`. +ENV _CE_CONDA='' \ + _CE_M='' \ + CONDA_DEFAULT_ENV=hga \ + CONDA_EXE=/opt/conda/bin/conda \ + CONDA_PREFIX=/opt/conda/envs/hga \ + CONDA_PREFIX_1=/opt/conda \ + CONDA_PROMPT_MODIFIER=(hga) \ + CONDA_PYTHON_EXE=/opt/conda/bin/python \ + CONDA_ROOT=/opt/conda \ + CONDA_SHLVL=2 \ + PATH="/opt/conda/envs/hga/bin:${PATH}" \ + SHLVL=1 + +# GDAL specific environment variables +ENV CPL_ZIP_ENCODING=UTF-8 \ + GDAL_DATA=/opt/conda/envs/hga/share/gdal \ + GSETTINGS_SCHEMA_DIR=/opt/conda/envs/hga/share/glib-2.0/schemas \ + GSETTINGS_SCHEMA_DIR_CONDA_BACKUP='' \ + PROJ_LIB=/opt/conda/envs/hga/share/proj + # Set entrypoint to run test script. ENTRYPOINT ["/home/tests/run_tests.sh"] diff --git a/version.txt b/version.txt index f2ae0b4..0b1f1ed 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.2.12 +1.2.13