Envvar support

Dockerfile

@@ -1,13 +1,15 @@
 FROM mambaorg/micromamba:1.5.8-alpine3.20
 
+VOLUME /data
 WORKDIR /app
 
 # Activate the conda environment during build process
 ARG MAMBA_DOCKERFILE_ACTIVATE=1
 
-COPY ./conda-lock.yml .
 
+# Install dependencies
 # NOTE: `-p` is important to install to the "base" env
+COPY ./conda-lock.yml .
 RUN micromamba install -y \
     -p /opt/conda \
     -f conda-lock.yml \

antarctica_today/__init__.py

@@ -1,6 +1,11 @@
 import os
 import sys
 
+from antarctica_today.config import Settings
+
+config = Settings()
+
+
 # IMPORTANT: If we don't specify this setting, then the projection we want to use will
 # be replaced with another (and this warning will be printed)!
 #

antarctica_today/config.py

@@ -0,0 +1,41 @@
+from functools import cached_property
+
+from pydantic import DirectoryPath, computed_field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from antarctica_today.constants.paths import REPO_DIR
+
+
+class Settings(BaseSettings):
+    """Configuration required to download brightness temperature data."""
+
+    model_config = SettingsConfigDict(
+        env_prefix="ANTARCTICA_TODAY_",
+        env_file=".env",
+        extra="ignore",
+    )
+
+    # New data directories:
+    # - Keep data included in the repo separate from runtime data. We don't want to require
+    #   a specific directory structure that may not work on every computer. For example, on
+    #   NSIDC VMs, we have limited direct storage, and need to use mounts to access larger
+    #   storage devices.
+    # - Use environment variables to enable override; in this case I think we only need one
+    #   for the root storage directory. Default to an in-repo storage location so if the
+    #   envvars are not populated, system pollution doesn't occur.
+    # - Migrate more things iteratively :)
+    STORAGE_BASEDIR: DirectoryPath = REPO_DIR
+
+    @computed_field  # type:ignore[misc]
+    @cached_property
+    def db_dir(self) -> DirectoryPath:
+        directory = self.STORAGE_BASEDIR / "database"
+        directory.mkdir(parents=True, exist_ok=True)
+        return directory
+
+    @computed_field  # type:ignore[misc]
+    @cached_property
+    def plots_dir(self) -> DirectoryPath:
+        directory = self.STORAGE_BASEDIR / "plots"
+        directory.mkdir(parents=True, exist_ok=True)
+        return directory

antarctica_today/constants/paths.py

@@ -10,27 +10,11 @@
 # The root of the Git repository (directory containing `.git`)
 REPO_DIR: Final = PACKAGE_DIR.parent
 
-# New data directories:
-# - Keep data included in the repo separate from runtime data. We don't want to require
-#   a specific directory structure that may not work on every computer. For example, on
-#   NSIDC VMs, we have limited direct storage, and need to use mounts to access larger
-#   storage devices.
-# - Use environment variables to enable override; in this case I think we only need one
-#   for the root storage directory. Default to an in-repo storage location so if the
-#   envvars are not populated, system pollution doesn't occur.
-# - Migrate more things iteratively :)
-_default_storage_dir = REPO_DIR
-STORAGE_DIR: Final = Path(
-    os.environ.get("ANTARCTICA_TODAY_STORAGE_DIR", _default_storage_dir)
-)
-DATA_DATABASE_DIR: Final = STORAGE_DIR / "database"
-# DATA_OUTPUT_DIR: Final = STORAGE_DIR / "output"
 
 # Legacy data directories
 DATA_DIR: Final = REPO_DIR / "data"
 DATA_QGIS_DIR: Final = REPO_DIR / "qgis"
 DATA_TB_DIR: Final = REPO_DIR / "Tb"
-DATA_PLOTS_DIR: Final = REPO_DIR / "plots"
 DATA_BASELINE_DATASETS_DIR: Final = REPO_DIR / "baseline_datasets"
 
 

antarctica_today/generate_antarctica_today_map.py

@@ -23,12 +23,11 @@
 from loguru import logger
 from osgeo import gdal
 
-from antarctica_today import read_NSIDC_bin_file, write_NSIDC_bin_to_gtif
+from antarctica_today import config, read_NSIDC_bin_file, write_NSIDC_bin_to_gtif
 from antarctica_today.compute_mean_climatology import (
     create_partial_year_melt_anomaly_tif,
     read_annual_melt_anomaly_tif,
 )
-from antarctica_today.constants.paths import DATA_DIR, DATA_PLOTS_DIR
 
 # import svgclip
 from antarctica_today.map_filedata import (
@@ -67,7 +66,7 @@ def main():
             #        for fmt in ("png", "pdf", "svg"):
             fig, ax = m.generate_annual_melt_map(
                 outfile_template=(
-                    DATA_PLOTS_DIR
+                    config.plots_dir
                     / "annual_maps_sum"
                     / f"R{region}_{year}-{year+1}_sum.{fmt}"
                 ),
@@ -83,7 +82,7 @@ def main():
 
             fig, ax = m.generate_anomaly_melt_map(
                 outfile_template=(
-                    DATA_PLOTS_DIR
+                    config.plots_dir
                     / "annual_maps_anomaly"
                     / f"R{region}_{year}-{year+1}_anomaly.{fmt}"
                 ),
@@ -97,7 +96,7 @@ def main():
 
             plt.close(fig)
 
-    # fig, ax = m.generate_anomaly_melt_map(DATA_PLOTS_DIR / "annual_maps_anomaly/R0_2021-2022.04.30_text.png",
+    # fig, ax = m.generate_anomaly_melt_map(config.plots_dir / "annual_maps_anomaly/R0_2021-2022.04.30_text.png",
     #                                       year=2021+1,
     #                                       dpi=300,
     #                                       include_scalebar=True,
@@ -106,12 +105,12 @@ def main():
     #                                       reset_picklefile=False)
 
     # fig, ax = m.generate_daily_melt_map(DATA_DIR / "v2.5/antarctica_melt_S3B_2010-2020_20200129/antarctica_melt_20100101_S3B_20210129.bin",
-    #                           outfile = DATA_PLOTS_DIR / "v2.5/daily_maps/20100101_daily.jpg", dpi=150)
+    #                           outfile = config.plots_dir / "v2.5/daily_maps/20100101_daily.jpg", dpi=150)
 
     # print (m._get_current_axes_position(ax))
 
     # for fmt in ("png", "svg"):
-    #     m.generate_annual_melt_map(outfile_template=DATA_PLOTS_DIR / "v2.5/annual_maps/R{1}_{0}-{3}." + fmt,
+    #     m.generate_annual_melt_map(outfile_template=config.plots_dir / "v2.5/annual_maps/R{1}_{0}-{3}." + fmt,
     #                                 region_number=0,
     #                                 year=2020,
     #                                 dpi=600,
@@ -121,7 +120,7 @@ def main():
     #                                 # include_current_date_label=True)
 
     #     # m.generate_anomaly_melt_map(year="all", reset_picklefile=True)
-    #     m.generate_anomaly_melt_map(outfile_template=DATA_PLOTS_DIR / "v2.5/anomaly_maps/R{1}_{0}-{3}." + fmt,
+    #     m.generate_anomaly_melt_map(outfile_template=config.plots_dir / "v2.5/anomaly_maps/R{1}_{0}-{3}." + fmt,
     #                                 year=2020,
     #                                 region_number=0,
     #                                 message_below_year="through 16 February,\n relative to 1990-2020",
@@ -130,7 +129,7 @@ def main():
     #                                 # reset_picklefile=True)
 
     # for melt_code in range(2,8+1):
-    #     m.generate_cumulative_melt_map(outfile_template = DATA_PLOTS_DIR / "v2.5/annual_maps/{0}_region{1}_level{2}.jpg",
+    #     m.generate_cumulative_melt_map(outfile_template = config.plots_dir / "v2.5/annual_maps/{0}_region{1}_level{2}.jpg",
     #                                    melt_code_threshold=melt_code,
     #                                    year="all")
     #                                    # year=2015)

antarctica_today/generate_plots_for_given_day.py

@@ -7,6 +7,7 @@
 import os
 import re
 import shutil
+from pathlib import Path
 
 import dateutil.parser
 import matplotlib.pyplot
@@ -105,7 +106,7 @@ def generate_maps_and_plots_for_a_date(
             region_num=region_num,
             gap_filled=True,
             dpi=dpi,
-            outfile=lineplot_outfile,
+            outfile=Path(lineplot_outfile),
         )
 
         # Close the current plots open in matplotlib. (Keeps them from accumulating.)

antarctica_today/main.py

@@ -7,6 +7,7 @@
 import argparse
 import datetime
 import os
+from pathlib import Path
 
 import click
 
@@ -20,7 +21,7 @@
 )
 
 
-def preprocessing_main():
+def preprocessing_main() -> None:
     """When we get new data (or new versions of the data), do all the things to get it ingested.
 
     1) Read all the .bin files and put them into the array picklefile
@@ -57,7 +58,7 @@ def preprocessing_main():
         compute_mean_climatology.create_annual_melt_anomaly_tif(year, gap_filled=True)
 
 
-def generate_all_plots_and_maps_main():
+def generate_all_plots_and_maps_main() -> None:
     """After all the preprocessing, re-generate all the plots and maps.
 
     4) Re-run the climatology & daily-melt plots for each year.
@@ -74,7 +75,7 @@ def generate_all_plots_and_maps_main():
             plot_daily_melt_and_climatology.plot_current_year_melt_over_baseline_stats(
                 datetime.datetime(year=year + 1, month=4, day=30),
                 region_num=region,
-                outfile=fname,
+                outfile=Path(fname),
             )
 
     # 5) Get a quick status check on the dates coverage.

antarctica_today/plot_daily_melt_and_climatology.py

@@ -7,6 +7,8 @@
 
 import datetime
 import os
+from pathlib import Path
+from typing import Optional
 
 import matplotlib as mpl
 import matplotlib.pyplot as plt
@@ -212,7 +214,7 @@ def plot_current_year_melt_over_baseline_stats(
     region_num=0,
     doy_start=(10, 1),
     doy_end=(4, 30),
-    outfile=None,
+    outfile: Optional[Path] = None,
     gap_filled=True,
     add_max_line=False,
     dpi=300,
@@ -365,7 +367,7 @@ def _plot_current_year_and_baseline(
     current_year_percents,
     fraction_x,
     region_num=0,
-    outfile=None,
+    outfile: Optional[Path] = None,
     gap_filled=True,
     add_max_line=False,
     dpi=300,
@@ -434,12 +436,14 @@ def _plot_current_year_and_baseline(
         # _add_region_area_at_bottom(fig, ax, region_number=region_num)
 
     if outfile:
+        outfile.parent.mkdir(parents=True, exist_ok=True)
+
         if gap_filled and os.path.split(outfile)[1].find("gap_filled") == -1:
             base, ext = os.path.splitext(outfile)
-            outfile = base + "_gap_filled" + ext
+            outfile = Path(base + "_gap_filled" + ext)
 
         logger.debug(f"Plotting {outfile}")
-        if os.path.splitext(outfile)[1].lower() == ".eps":
+        if outfile.suffix.lower() == ".eps":
             fig.savefig(outfile, dpi=dpi, format="eps")
         else:
             fig.savefig(outfile, dpi=dpi)
@@ -1266,7 +1270,7 @@ def special_figure_REG5_FEB_APR_2022(outfile):
             plot_current_year_melt_over_baseline_stats(
                 current_date=datetime.datetime(year + 1, 4, 30),
                 region_num=region_num,
-                outfile=fname,
+                outfile=Path(fname),
                 dpi=1200,
                 add_max_line=False,
                 gap_filled=True,

antarctica_today/tb_file_data.py

@@ -9,11 +9,10 @@
 
 import os
 
+from antarctica_today import config
 from antarctica_today.constants.paths import (
     DATA_BASELINE_DATASETS_DIR,
-    DATA_DATABASE_DIR,
     DATA_DIR,
-    DATA_PLOTS_DIR,
     DATA_TB_DIR,
 )
 
@@ -63,6 +62,7 @@ def recurse_directory(directory, ignore="thresholds", target=".bin", sorted=True
 
 NSIDC_0080_file_dir = DATA_TB_DIR / "nsidc-0080"
 
+DATA_DATABASE_DIR = config.db_dir
 # TODO: More consistent names; "model_results" doesn't tell us much about where these
 # files live. These live in external storage, but others live in repo (for now). Pick
 # either "dir" or "folder". Do we need "v3" in the filenames? The code only supports one
@@ -89,7 +89,7 @@ def recurse_directory(directory, ignore="thresholds", target=".bin", sorted=True
 
 model_results_v3_dir = DATA_DIR
 model_results_dir = model_results_v3_dir / "daily_melt_bin_files"
-model_results_plot_directory = DATA_PLOTS_DIR
+model_results_plot_directory = config.plots_dir
 # output_tifs_directory = os.path.join(model_results_v3_dir, "sample_results")
 outputs_annual_tifs_directory = os.path.join(model_results_v3_dir, "annual_sum_geotifs")
 outputs_annual_plots_directory = os.path.join(

compose.yml

@@ -3,8 +3,12 @@ services:
     image: "nsidc/antarctica_today:${ANTARCTICA_TODAY_VERSION:-latest}"
     user: "root"
     volumes:
-      - "./Tb/:/app/Tb/"
       - "./data/:/app/data/"
-      - "./database/:/app/database/"
       - "./plots/:/app/plots/"
       - "./baseline_datasets/:/app/baseline_datasets/:ro"
+      - "${ANTARCTICA_TODAY_STORAGE_BASEDIR:-/tmp/antarctica_today}:/data"
+    environment:
+      # NOTE: null value indicates passthrough from host.
+      EARTHDATA_USERNAME: null
+      EARTHDATA_PASSWORD: null
+      ANTARCTICA_TODAY_STORAGE_BASEDIR: "/data"