Merge pull request #224 from nasa/release/1.3.0

Release/1.3.0
nasa · Jul 15, 2024 · 2c845e1 · 2c845e1
2 parents b27270c + febcf32
commit 2c845e1
Show file tree

Hide file tree

Showing 52 changed files with 3,390 additions and 685 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -0,0 +1,5 @@
+flags:
+  unittests:
+    carryforward: false
+  integration:
+    carryforward: false
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -4,11 +4,19 @@ updates:
     directory: "/"
     schedule:
       interval: "monthly"
+    groups:
+      pip-dependencies:
+        patterns:
+          - "*"
     # Raise pull requests for version updates
     # to pip against the `develop` branch
     target-branch: "develop"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "monthly"
+    groups:
+      gha-dependencies:
+        patterns:
+          - "*"
     target-branch: "develop"
diff --git a/.github/workflows/build-pipeline.yml b/.github/workflows/build-pipeline.yml
@@ -5,7 +5,7 @@ name: Build
 on:
   # Triggers the workflow on push events
   push:
-    branches: [ develop, release/**, main, feature/**, issue/**, issues/** ]
+    branches: [ develop, release/**, main, feature/**, issue/**, issues/** , docs/**]
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
@@ -51,7 +51,8 @@ jobs:
         if: |
           ${{ startsWith(github.ref, 'refs/heads/issue') }}      ||
           ${{ startsWith(github.ref, 'refs/heads/dependabot/') }} ||
-          ${{ startsWith(github.ref, 'refs/heads/feature/') }}
+          ${{ startsWith(github.ref, 'refs/heads/feature/') }} ||
+          ${{ startsWith(github.ref, 'refs/heads/docs/') }}
         run: |
           new_ver="${{ steps.get-version.outputs.current_version }}+$(git rev-parse --short ${GITHUB_SHA})"
           poetry version $new_ver
@@ -169,7 +170,7 @@ jobs:
       - name: Build and push Docker image
         if: ${{ !startsWith(github.ref, 'refs/heads/main/') }}
         id: docker-push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: Dockerfile

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -0,0 +1,64 @@
+name: Integration Tests
+
+on:
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    secrets:
+      DEK_EDL_USER:
+        required: true
+      DEK_EDL_PASSWORD:
+        required: true
+      codecov_token:
+        required: true
+  push:
+    branches:
+      - main
+      - develop
+      - release/**
+      - feature/**
+
+# When this workflow is queued, automatically cancel any previous running
+# or pending jobs from the same branch
+concurrency:
+  group: integration-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  POETRY_VERSION: "1.3.2"
+  PYTHON_VERSION: "3.10"
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Retrieve repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Set up Poetry
+        uses: abatilo/actions-poetry@v3.0.0
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+
+      - name: Install package
+        run: poetry install --with=integration --without harmony
+
+      - name: Test
+        env:
+          EDL_USER: ${{ secrets.DEK_EDL_USER }}
+          EDL_PASSWORD: ${{ secrets.DEK_EDL_PASSWORD }}
+        run: |
+          scripts/create-netrc
+          poetry run pytest --cov=concatenator --cov-report=xml tests/integration
+
+      - name: Upload coverage
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: true
+          flags: integration
diff --git a/.github/workflows/pr-rtd-link.yml b/.github/workflows/pr-rtd-link.yml
@@ -0,0 +1,21 @@
+# The ReadTheDocs preview link is "hidden" within the GitHub "Checks"
+# interface. For users who don't know this, finding the preview link may be
+# very difficult or frustrating. This workflow makes the link more
+# findable by updating PR descriptions to include it.
+name: "Add ReadTheDocs preview link to PR descriptions"
+
+on:
+  pull_request_target:
+    types:
+      - opened
+
+permissions:
+  pull-requests: "write"
+
+jobs:
+  autolink-rtd-previews:
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: "readthedocs/actions/preview@v1"
+        with:
+          project-slug: "stitchee"
diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
@@ -1,9 +1,10 @@
+# A reusable workflow to build and run the unit test suite
+#
 # This workflow will install Python dependencies, run tests,
 # and report test results and code coverage as artifacts. It will
 # be called by the workflow that run tests against new PRs and as
 # a first step in the workflow that publishes new Docker images.
-
-name: A reusable workflow to build and run the unit test suite
+name: Unit Tests
 
 on:
   workflow_call:
@@ -17,7 +18,7 @@ env:
   PYTHON_VERSION: "3.10"
 
 jobs:
-  build_and_test:
+  unit-tests:
     runs-on: ubuntu-latest
 
     steps:
@@ -35,18 +36,18 @@ jobs:
           poetry-version: ${{ env.POETRY_VERSION }}
 
       - name: Install package
-        run: poetry install
+        run: poetry install --with=harmony --without integration
 
       - name: Run linting
         run: |
           poetry run ruff check concatenator
 
       - name: Run tests and collect coverage
-        run: poetry run pytest --cov=concatenator tests/unit/test_dataset_and_group_handling.py --cov-report=xml
-        # TODO: expand tests to include full concatenation runs, i.e., not only test_dataset_and_group_handling.py
+        run: poetry run pytest --cov=concatenator --cov-report=xml tests/unit
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
+          flags: unittests
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,4 +1,10 @@
 ---
+ci:
+  autoupdate_schedule: "monthly" # Like dependabot
+  autoupdate_commit_msg: "chore: update pre-commit hooks"
+  autoupdate_branch: "develop"
+  autofix_prs: false # Comment "pre-commit.ci autofix" on a PR to trigger
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
@@ -15,7 +21,7 @@ repos:
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.4.2'
+    rev: 'v0.4.4'
     hooks:
       - id: ruff
         args: [ "--fix" ]

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -0,0 +1,27 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.11"
+  jobs:
+    post_create_environment:
+      # Install poetry
+      # https://python-poetry.org/docs/#installing-manually
+      - pip install poetry
+      - poetry self add poetry-dynamic-versioning
+      # Tell poetry to not use a virtual environment
+      # - poetry config virtualenvs.create false
+    post_install:
+      # Install dependencies with 'docs' dependency group
+      # https://python-poetry.org/docs/managing-dependencies/#dependency-groups
+      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with=dev
+
+mkdocs:
+  configuration: mkdocs.yml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,63 +1,86 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+The format is based on [Common Changelog](https://common-changelog.org/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.2.1]
+## [1.3.0] - 2024-07-11
 
-### Added
 ### Changed
-  - [Issue #183](https://github.com/nasa/stitchee/issues/183): Update UMM-S record ID
-### Deprecated
-### Removed
+
+- Group dependabot updates into one PR ([#206](https://github.com/nasa/stitchee/issues/206))([**@danielfromearth**](https://github.com/danielfromearth))
+- Increase continuous integration/unit test coverage ([#208](https://github.com/nasa/stitchee/issues/208))([**@danielfromearth**](https://github.com/danielfromearth))
+- Use time variable instead of concat dim for ordering datasets ([#198](https://github.com/nasa/stitchee/issues/198))([**@danielfromearth**](https://github.com/danielfromearth), [**@ank1m**](https://github.com/ank1m))
+- Update `CHANGELOG.md` to follow Common Changelog conventions ([#226](https://github.com/nsidc/earthaccess/pull/226))([**@danielfromearth**](https://github.com/danielfromearth))
+
+### Added
+
+- Add readthedocs documentation build ([#133](https://github.com/nasa/stitchee/issues/133))([**@danielfromearth**](https://github.com/danielfromearth))
+- Add arguments for temporary file copies and overwriting output file in main stitchee function ([#185](https://github.com/nasa/stitchee/issues/185))([**@danielfromearth**](https://github.com/danielfromearth))
+- Add a group delimiter argument ([#181](https://github.com/nasa/stitchee/issues/181))([**@danielfromearth**](https://github.com/danielfromearth))
+- Add an integration test that runs stitchee on files first subsetted by the operational Harmony subsetter ([#134](https://github.com/nasa/stitchee/issues/134))([**@danielfromearth**](https://github.com/danielfromearth))
+- Add page about the SAMBAH service chain to the Readthedocs documentation ([#194](https://github.com/nasa/stitchee/issues/194))([**@danielfromearth**](https://github.com/danielfromearth))
+- Add autoupdate schedule for pre-commit ([#193](https://github.com/nasa/stitchee/issues/193))([**@danielfromearth**](https://github.com/danielfromearth))
+
 ### Fixed
 
+- Fix integration test failure ([#204](https://github.com/nasa/stitchee/issues/204))([**@danielfromearth**](https://github.com/danielfromearth))
+
+
+## [1.2.1]
+
+### Changed
+
+- Update UMM-S record ID ([#183](https://github.com/nasa/stitchee/issues/183))([**@danielfromearth**](https://github.com/danielfromearth), [**@ank1m**](https://github.com/ank1m))
+
 ## [1.2.0]
 
-### Added
-  - [Issue #170](https://github.com/nasa/stitchee/issues/170): Add PyPI badges to readme
 ### Changed
-  - [Issue #153](https://github.com/nasa/stitchee/issues/153): propagate first empty granule if all input files are empty
-  - [Issue #168](https://github.com/nasa/stitchee/issues/168): remove compression for string array of small size
-### Deprecated
-### Removed
+
+- Propagate first empty granule if all input files are empty ([#153](https://github.com/nasa/stitchee/issues/153))([**@ank1m**](https://github.com/ank1m), [**@danielfromearth**](https://github.com/danielfromearth))
+- Remove compression for string array of small size ([#168](https://github.com/nasa/stitchee/issues/168))([**@ank1m**](https://github.com/ank1m), [**@danielfromearth**](https://github.com/danielfromearth))
+
+### Added
+
+- Add PyPI badges to readme ([#170](https://github.com/nasa/stitchee/issues/170))([**@danielfromearth**](https://github.com/danielfromearth))
+
 ### Fixed
-  - [Pull #177](https://github.com/nasa/stitchee/pull/177): Resolve linting error
+
+- Resolve linting error ([#177](https://github.com/nasa/stitchee/pull/177))([**@danielfromearth**](https://github.com/danielfromearth))
 
 ## [1.1.0]
 
 ### Added
-  - [Pull #164](https://github.com/nasa/stitchee/pull/164): Add code coverage to CI pipeline
-### Changed
-### Deprecated
-### Removed
-### Fixed
+
+- Add code coverage to CI pipeline ([#164](https://github.com/nasa/stitchee/pull/164))([**@danielfromearth**](https://github.com/danielfromearth))
 
 ## [1.0.0]
 
-### Added
-  - [Pull #1](https://github.com/danielfromearth/stitchee/pull/1): An initial GitHub Actions workflow
-  - [Issue #8](https://github.com/danielfromearth/stitchee/issues/8): Create working Docker image
-  - [Issue #10](https://github.com/danielfromearth/stitchee/issues/10): Add code necessary to communicate with Harmony
-  - [Issue #49](https://github.com/danielfromearth/stitchee/issues/49): More CLI arguments for finer control of concatenation method
-  - [Pull #99](https://github.com/danielfromearth/stitchee/pull/99): Add Docker build steps to GitHub Actions workflow
-  - [Pull #113](https://github.com/danielfromearth/stitchee/pull/113): Add history attributes
-  - [Pull #115](https://github.com/danielfromearth/stitchee/pull/115): Add readme badges
-  - [Pull #116](https://github.com/danielfromearth/stitchee/pull/116): Add tutorial Jupyter notebook
-  - [Pull #128](https://github.com/danielfromearth/stitchee/pull/116): Create toy netCDFs in testing suite
 ### Changed
-  - [Pull #12](https://github.com/danielfromearth/stitchee/pull/12): Changed name to "stitchee"
-  - [Pull #15](https://github.com/danielfromearth/stitchee/pull/15): Use ruff+black chain for pre-commit lint & format
-  - [Issue #45](https://github.com/danielfromearth/stitchee/issues/45): Rename CLI argument to clarify temporary copying behavior
-  - [Issue #44](https://github.com/danielfromearth/stitchee/issues/44): Concatenation dimension CLI argument is required but isn't listed as such in the help message
-  - [Issue #81](https://github.com/danielfromearth/stitchee/issues/81): Remove `nco` related code
-  - [Pull #129](https://github.com/danielfromearth/stitchee/pull/129): Sort according to extend dimension
-  - [Pull #152](https://github.com/danielfromearth/stitchee/pull/152): Consider empty a netCDF with only singleton null-values
-  - [Pull #157](https://github.com/danielfromearth/stitchee/pull/157): Update CI pipeline
-  - [Pull #158](https://github.com/danielfromearth/stitchee/pull/158): Add pypi publishing steps to CI pipeline
-### Deprecated
-### Removed
+
+- Change name to "stitchee" ([#12](https://github.com/danielfromearth/stitchee/pull/12))([**@danielfromearth**](https://github.com/danielfromearth))
+- Use ruff+black chain for pre-commit lint & format ([#15](https://github.com/danielfromearth/stitchee/pull/15))([**@danielfromearth**](https://github.com/danielfromearth))
+- Rename CLI argument to clarify temporary copying behavior ([#45](https://github.com/danielfromearth/stitchee/issues/45))([**@danielfromearth**](https://github.com/danielfromearth))
+- Remove `nco` related code ([#83](https://github.com/nasa/stitchee/pull/83))([**@danielfromearth**](https://github.com/danielfromearth))
+- Sort according to extend dimension ([#129](https://github.com/danielfromearth/stitchee/pull/129))([**@danielfromearth**](https://github.com/danielfromearth))
+- Consider empty a netCDF with only singleton null-values ([#152](https://github.com/danielfromearth/stitchee/pull/152))([**@danielfromearth**](https://github.com/danielfromearth), [**@ank1m**](https://github.com/ank1m))
+- Update CI pipeline ([#157](https://github.com/danielfromearth/stitchee/pull/157))([**@danielfromearth**](https://github.com/danielfromearth))
+- Add pypi publishing steps to CI pipeline ([#158](https://github.com/danielfromearth/stitchee/pull/158))([**@danielfromearth**](https://github.com/danielfromearth))
+
+### Added
+
+- An initial GitHub Actions workflow ([#1](https://github.com/danielfromearth/stitchee/pull/1))([**@danielfromearth**](https://github.com/danielfromearth))
+- Create working Docker image ([#40](https://github.com/nasa/stitchee/pull/40))([**@danielfromearth**](https://github.com/danielfromearth))
+- Add code necessary to communicate with Harmony ([#40](https://github.com/nasa/stitchee/pull/40))([**@danielfromearth**](https://github.com/danielfromearth))
+- More CLI arguments for finer control of concatenation method ([#50](https://github.com/nasa/stitchee/pull/50))([**@danielfromearth**](https://github.com/danielfromearth))
+- Add Docker build steps to GitHub Actions workflow ([#99](https://github.com/danielfromearth/stitchee/pull/99), [#108](https://github.com/nasa/stitchee/pull/108))([**@danielfromearth**](https://github.com/danielfromearth))
+- Add history attributes ([#113](https://github.com/danielfromearth/stitchee/pull/113))([**@danielfromearth**](https://github.com/danielfromearth), [**@ank1m**](https://github.com/ank1m))
+- Add readme badges ([#115](https://github.com/danielfromearth/stitchee/pull/115))([**@danielfromearth**](https://github.com/danielfromearth))
+- Add tutorial Jupyter notebook ([#116](https://github.com/danielfromearth/stitchee/pull/116))([**@ank1m**](https://github.com/ank1m), [**@danielfromearth**](https://github.com/danielfromearth))
+- Create toy netCDFs in testing suite ([#128](https://github.com/danielfromearth/stitchee/pull/128))([**@danielfromearth**](https://github.com/danielfromearth))
+
 ### Fixed
-- [Pull #4](https://github.com/danielfromearth/stitchee/pull/4): Error with TEMPO ozone profile data because of duplicated dimension names
-- [Pull #133](https://github.com/danielfromearth/stitchee/pull/133): Fix conflicting dimensions on record dimension sorting
+
+- Error with TEMPO ozone profile data because of duplicated dimension names ([#4](https://github.com/danielfromearth/stitchee/pull/4))([**@danielfromearth**](https://github.com/danielfromearth))
+- Fix conflicting dimensions on record dimension sorting ([#136](https://github.com/danielfromearth/stitchee/pull/136))([**@danielfromearth**](https://github.com/danielfromearth), [**@ank1m**](https://github.com/ank1m))
+- Concatenation dimension CLI argument is required but isn't listed as such in the help message ([#44](https://github.com/danielfromearth/stitchee/issues/44))([**@danielfromearth**](https://github.com/danielfromearth))
diff --git a/Dockerfile b/Dockerfile
@@ -39,7 +39,7 @@ COPY --chown=dockeruser $DIST_PATH $DIST_PATH
 
 #install poetry as root
 RUN poetry config virtualenvs.create false
-RUN poetry install --only main
+RUN poetry install --with harmony --without integration
 
 USER dockeruser
 COPY --chown=dockeruser ./docker-entrypoint.sh docker-entrypoint.sh