Add main COOPS API for metadata

.pre-commit-config.yaml

@@ -11,6 +11,7 @@ repos:
     rev: "v4.5.0"
     hooks:
       - id: "check-added-large-files"
+        exclude: "tests/cassettes/.*"
       - id: "check-ast"
       - id: "check-byte-order-marker"
       - id: "check-docstring-first"

searvey/coops.py

@@ -6,6 +6,7 @@
 
 import json
 import logging
+import warnings
 from abc import ABC
 from abc import abstractmethod
 from datetime import datetime
@@ -14,6 +15,7 @@
 from pathlib import Path
 from typing import Any
 from typing import Dict
+from typing import Optional
 from typing import Union
 
 import geopandas
@@ -32,6 +34,8 @@
 from shapely.geometry import Polygon
 from xarray import Dataset
 
+from .utils import get_region
+
 
 logger = logging.getLogger(__name__)
 
@@ -53,6 +57,11 @@
     DISCONTINUED = "discontinued"
 
 
+class StationMetadataSource(Enum):
+    MAIN = "main"
+    NWS = "nws"
+
+
 class Station(ABC):
     """
     abstraction of a specific data station
@@ -695,6 +704,7 @@
     [71 rows x 6 columns]
     """
 
+    warnings.warn("Using older API, will be removed in the future!", DeprecationWarning)
     tables = __coops_stations_html_tables()
 
     status_tables = {
@@ -775,7 +785,9 @@
 
     stations = pandas.concat(
         (
-            active_stations[~active_stations.index.isin(discontinued_stations.index)].drop(columns="removed"),  # fmt: skip
+            active_stations[~active_stations.index.isin(discontinued_stations.index)].drop(
+                columns='removed'
+            ),  # fmt: skip
             discontinued_stations,
         )
     )
@@ -897,4 +909,98 @@
     return xarray.combine_nested(station_data, concat_dim="nos_id")
 
 
-get_coops_stations = coops_stations_within_region
+def normalize_coops_stations(df: pandas.DataFrame) -> geopandas.GeoDataFrame:
+    df = (
+        df.rename(
+            columns={
+                "id": "nos_id",
+                "shefcode": "nws_id",
+                "lat": "lat",
+                "lng": "lon",
+                "details.removed": "removed",
+            },
+        )
+        .astype(
+            {
+                "nos_id": numpy.int32,
+                "nws_id": "string",
+                "lon": numpy.float32,
+                "lat": numpy.float32,
+                "state": "string",
+                "name": "string",
+                "removed": "datetime64[ns]",
+            },
+        )
+        .set_index("nos_id")[
+            [
+                "nws_id",
+                "name",
+                "state",
+                "lon",
+                "lat",
+                "removed",
+            ]
+        ]
+    )
+    df["status"] = StationStatus.ACTIVE.value
+    df.loc[~df.removed.isna(), "status"] = StationStatus.DISCONTINUED.value
+    gdf = geopandas.GeoDataFrame(
+        data=df,
+        geometry=geopandas.points_from_xy(df.lon, df.lat, crs="EPSG:4326"),
+    )
+    return gdf
+
+
+@lru_cache(maxsize=1)
+def _get_coops_stations() -> geopandas.GeoDataFrame:
+    """
+    Return COOPS station metadata from: COOPS main API
+
+    :return: ``geopandas.GeoDataFrame`` with the station metadata
+    """
+
+    url_active = "https://api.tidesandcurrents.noaa.gov/mdapi/prod/webapi/stations.json?expand=details"
+    url_historic = "https://api.tidesandcurrents.noaa.gov/mdapi/prod/webapi/stations.json?type=historicwl&expand=details"
+
+    df_active = pandas.read_json(url_active)
+    df_active = pandas.json_normalize(df_active["stations"])
+
+    df_historic = pandas.read_json(url_historic)
+    df_historic = pandas.json_normalize(df_historic["stations"])
+    df_historic = df_historic[~df_historic.id.isin(df_active.id)]
+
+    df = pandas.concat((df_active, df_historic))
+
+    coops_stations = normalize_coops_stations(df)
+    return coops_stations
+
+
+def get_coops_stations(
+    region: Optional[Union[Polygon, MultiPolygon]] = None,
+    lon_min: Optional[float] = None,
+    lon_max: Optional[float] = None,
+    lat_min: Optional[float] = None,
+    lat_max: Optional[float] = None,
+    metadata_source: Union[StationMetadataSource, str] = "nws",
+) -> geopandas.GeoDataFrame:
+    md_src = StationMetadataSource(metadata_source)
+
+    region = get_region(
+        region=region,
+        lon_min=lon_min,
+        lon_max=lon_max,
+        lat_min=lat_min,
+        lat_max=lat_max,
+        symmetric=True,
+    )
+
+    if md_src == StationMetadataSource.MAIN:
+        coops_stations = _get_coops_stations()
+        if region:
+            coops_stations = coops_stations[coops_stations.within(region)]
+    elif md_src == StationMetadataSource.NWS:
+        coops_stations = coops_stations_within_region(region)
+    else:
+        raise ValueError("Unknown metadata source specified!")
+
+    return coops_stations

searvey/stations.py

@@ -4,7 +4,6 @@
 from enum import Enum
 
 import geopandas as gpd
-import numpy as np
 import pandas as pd
 from shapely.geometry import MultiPolygon
 from shapely.geometry import Polygon
@@ -92,20 +91,17 @@ def _get_ioc_stations(
 def _get_coops_stations(
     region: Polygon | MultiPolygon | None = None,
 ) -> gpd.GeoDataFrame:
-    coops_gdf = coops.coops_stations_within_region(region=region)
+    coops_gdf = coops.get_coops_stations(region=region, metadata_source="main")
     coops_gdf = coops_gdf.assign(
         provider=Provider.COOPS.value,
         provider_id=coops_gdf.index,
-        country=np.where(coops_gdf.state.str.len() > 0, "USA", None),  # type: ignore[call-overload]
+        country=coops_gdf.state.where(coops_gdf.state.str.len() != 2, "USA"),
         location=coops_gdf[["name", "state"]].agg(", ".join, axis=1),
         lon=coops_gdf.geometry.x,
         lat=coops_gdf.geometry.y,
         is_active=coops_gdf.status == "active",
         start_date=pd.NaT,
-        last_observation=coops_gdf[coops_gdf.status == "discontinued"]
-        .removed.str[:18]
-        .apply(pd.to_datetime)
-        .dt.tz_localize("UTC"),
+        last_observation=coops_gdf[coops_gdf.status == "discontinued"].removed.dt.tz_localize("UTC"),
     )[STATIONS_COLUMNS]
     return coops_gdf