Update documentation

.buildinfo

@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: 1025286996b58bc9cf006461d359d7b4
+tags: 645f666f9bcd5a90fca523b33c5a78b7

_downloads/074212d9cab0d4da5bc9827aca373ee9/grid_loading_example.ipynb

@@ -0,0 +1,241 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "f87ea1e0",
+   "metadata": {},
+   "source": [
+    "# NWM Gridded Data Loading"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "38f2f9f4-5294-4f9b-971a-931b91da8f85",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Import the required packages.\n",
+    "\n",
+    "# Run this if TEEHR is not installed\n",
+    "# import sys\n",
+    "# sys.path.insert(0, \"../../src\")\n",
+    "\n",
+    "import os\n",
+    "\n",
+    "from dask.distributed import Client\n",
+    "from pathlib import Path\n",
+    "\n",
+    "import teehr.loading.nwm.nwm_grids as tlg\n",
+    "\n",
+    "import teehr.utilities.generate_weights as gw\n",
+    "from teehr.loading.nwm.const import CONUS_NWM_WKT"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Specify input variables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "76e363e7-a3e0-427f-811f-a6298b93536f",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Set some notebook variables to point to the relevant study files.\n",
+    "TEMP_GEO_DIR = Path(Path.home(), \"temp/geo\")\n",
+    "TEMP_GEO_DIR.mkdir(exist_ok=True, parents=True)\n",
+    "\n",
+    "# Generate weights\n",
+    "# fetch \"https://storage.googleapis.com/national-water-model/nwm.20220101/forcing_short_range/nwm.t00z.short_range.forcing.f001.conus.nc\"\n",
+    "GRID_TEMPLATE_FILE = Path(TEMP_GEO_DIR, \"nwm.t00z.short_range.forcing.f001.conus.nc\")\n",
+    "\n",
+    "# fetch \"https://nextgen-hydrofabric.s3.amazonaws.com/v1.2/nextgen_03S.gpkg\"\n",
+    "ZONE_GEO_FILE = Path(TEMP_GEO_DIR, \"nextgen_03S.gpkg\")\n",
+    "ZONAL_WEIGHTS_FILEPATH = Path(TEMP_GEO_DIR, \"nextgen_03S_weights.parquet\")\n",
+    "UNIQUE_ZONE_ID = \"id\"\n",
+    "\n",
+    "# NWM\n",
+    "CONFIGURATION = \"forcing_short_range\"  # forcing_short_range, forcing_analysis_assim, forcing_medium_range\n",
+    "OUTPUT_TYPE = \"forcing\"\n",
+    "VARIABLE_NAME = \"RAINRATE\"\n",
+    "\n",
+    "START_DATE = \"2020-12-18\"\n",
+    "INGEST_DAYS = 1\n",
+    "\n",
+    "JSON_DIR = Path(Path.home(), \"temp/parquet/jsons/\")\n",
+    "OUTPUT_DIR = Path(Path.home(), \"temp/parquet\")\n",
+    "\n",
+    "NWM_VERSION = \"nwm22\"  # Currently accepts \"nwm22\" or \"nwm30\"\n",
+    "                       # Use \"nwm22\" for dates prior to 09-19-2023\n",
+    "\n",
+    "DATA_SOURCE = \"GCS\"    # Specifies the remote location from which to fetch the data\n",
+    "                       # (\"GCS\", \"NOMADS\", \"DSTOR\")\n",
+    "\n",
+    "KERCHUNK_METHOD = \"auto\"  # When data_source = \"GCS\", specifies the preference in creating Kerchunk reference json files.\n",
+    "                          # \"local\" - always create new json files from netcdf files in GCS and save locally, if they do not already exist\n",
+    "                          # \"remote\" - read the CIROH pre-generated jsons from s3, ignoring any that are unavailable\n",
+    "                          # \"auto\" - read the CIROH pre-generated jsons from s3, and create any that are unavailable, storing locally\n",
+    "\n",
+    "CONCAT_DIMS = [\"time\"]  # \"reference_time\"\n",
+    "T_MINUS = [0, 1, 2]  # Only used if an assimilation run is selected\n",
+    "IGNORE_MISSING_FILE = True  # If True, the missing file(s) will be skipped and the process will resume\n",
+    "                            # If False, TEEHR will fail if a missing NWM file is encountered\n",
+    "OVERWRITE_OUTPUT = True  # If True, existing output files will be overwritten\n",
+    "                         # If False (default), existing files are retained"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Fetch a template forcing netCDF file"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "53724249-5fbe-4fa4-8fd9-96a3238d16b3",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "!wget -O /home/jovyan/temp/geo/nwm.t00z.short_range.forcing.f001.conus.nc \\\n",
+    "https://storage.googleapis.com/national-water-model/nwm.20220101/forcing_short_range/nwm.t00z.short_range.forcing.f001.conus.nc"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Fetch some example polygons (nextgen divides)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4a717b43",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "!wget -O /home/jovyan/temp/geo/nextgen_03S.gpkg https://lynker-spatial.s3.amazonaws.com/v20/gpkg/nextgen_03S.gpkg"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Start a local dask cluster"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "30f8283c",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "n_workers = max(os.cpu_count() - 1, 1)\n",
+    "client = Client(n_workers=n_workers)\n",
+    "client"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Generate the weights file"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0ac3c383-9aaf-4fd7-8c33-461ff5e87bb7",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "gw.generate_weights_file(\n",
+    "    zone_polygon_filepath=ZONE_GEO_FILE,\n",
+    "    template_dataset=GRID_TEMPLATE_FILE,\n",
+    "    variable_name=VARIABLE_NAME,\n",
+    "    output_weights_filepath=ZONAL_WEIGHTS_FILEPATH,\n",
+    "    crs_wkt=CONUS_NWM_WKT,\n",
+    "    unique_zone_id=UNIQUE_ZONE_ID,\n",
+    "    layer=\"divides\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Fetch the gridded data summarized to the polygons"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7b77bd47",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "tlg.nwm_grids_to_parquet(\n",
+    "    configuration=CONFIGURATION,\n",
+    "    output_type=OUTPUT_TYPE,\n",
+    "    variable_name=VARIABLE_NAME,\n",
+    "    start_date=START_DATE,\n",
+    "    ingest_days=INGEST_DAYS,\n",
+    "    zonal_weights_filepath=ZONAL_WEIGHTS_FILEPATH,\n",
+    "    json_dir=JSON_DIR,\n",
+    "    output_parquet_dir=OUTPUT_DIR,\n",
+    "    nwm_version=NWM_VERSION,\n",
+    "    data_source=DATA_SOURCE,\n",
+    "    kerchunk_method=KERCHUNK_METHOD,\n",
+    "    t_minus_hours=T_MINUS,\n",
+    "    ignore_missing_file=IGNORE_MISSING_FILE,\n",
+    "    overwrite_output=OVERWRITE_OUTPUT\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}