fix: backend monitoring changes (#371)

Changes to observability in backend APIs:

- Include monitoring for STAC API endpoints
- service dimension change for ingest api (incorrectly had
"raster-api"), add monitoring to ingest handler
- include path params and post body in logs
- include stage dimension in metrics to distinguish between instances
(i.e. dev, staging, etc.)
- include metrics for each api path (i.e. collections/,
collections/{collection_id}, mosaic/register, etc.

## Testing

- deployed to `monitor` veda-backend stack and created tables in
Grafana. See[
ticket](#349) for
details.
- Confirmed logs and metrics are being sent for raster, stac and ingest
APIs

app.py

@@ -122,6 +122,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
 
 # ingestor config requires references to other resources, but can be shared between ingest api and bulk ingestor
 ingestor_config = ingest_config(
+    stage=veda_app_settings.stage_name(),
     stac_db_security_group_id=db_security_group.security_group_id,
     stac_api_url=stac_api.stac_api.url,
     raster_api_url=raster_api.raster_api.url,

ingest_api/runtime/handler.py

@@ -4,5 +4,14 @@
 
 from mangum import Mangum
 from src.main import app
+from src.monitoring import logger, metrics, tracer
 
 handler = Mangum(app, lifespan="off", api_gateway_base_path=app.root_path)
+
+# Add tracing
+handler.__name__ = "handler"  # tracer requires __name__ to be set
+handler = tracer.capture_lambda_handler(handler)
+# Add logging
+handler = logger.inject_lambda_context(handler, clear_state=True)
+# Add metrics last to properly flush metrics.
+handler = metrics.log_metrics(handler, capture_cold_start_metric=True)

ingest_api/runtime/src/main.py

@@ -10,7 +10,7 @@
 from src.doc import DESCRIPTION
 from src.monitoring import LoggerRouteHandler, logger, metrics, tracer
 
-from fastapi import APIRouter, Depends, FastAPI, HTTPException
+from fastapi import Depends, FastAPI, HTTPException
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
 from fastapi.security import OAuth2PasswordRequestForm
@@ -32,9 +32,9 @@
         "clientId": settings.client_id,
         "usePkceWithAuthorizationCodeGrant": True,
     },
-    router=APIRouter(route_class=LoggerRouteHandler),
 )
 
+app.router.route_class = LoggerRouteHandler
 
 collection_publisher = CollectionPublisher()
 item_publisher = ItemPublisher()

ingest_api/runtime/src/monitoring.py

@@ -1,40 +1,55 @@
 """Observability utils"""
+import json
 from typing import Callable
 
-from aws_lambda_powertools import Logger, Metrics, Tracer
+from aws_lambda_powertools import Logger, Metrics, Tracer, single_metric
 from aws_lambda_powertools.metrics import MetricUnit  # noqa: F401
+from src.config import settings
 
 from fastapi import Request, Response
 from fastapi.routing import APIRoute
 
-logger: Logger = Logger(service="raster-api", namespace="veda-backend")
-metrics: Metrics = Metrics(service="raster-api", namespace="veda-backend")
+logger: Logger = Logger(service="ingest-api", namespace="veda-backend")
+metrics: Metrics = Metrics(namespace="veda-backend")
+metrics.set_default_dimensions(environment=settings.stage, service="ingest-api")
 tracer: Tracer = Tracer()
 
 
 class LoggerRouteHandler(APIRoute):
-    """Extension of base APIRoute to add context to log statements, as well as record usage metricss"""
+    """Extension of base APIRoute to add context to log statements, as well as record usage metrics"""
 
     def get_route_handler(self) -> Callable:
         """Overide route handler method to add logs, metrics, tracing"""
         original_route_handler = super().get_route_handler()
 
         async def route_handler(request: Request) -> Response:
             # Add fastapi context to logs
+            body = await request.body()
+            try:
+                body_json = json.loads(body)
+            except json.decoder.JSONDecodeError:
+                body_json = None
             ctx = {
                 "path": request.url.path,
+                "path_params": request.path_params,
+                "body": body_json,
                 "route": self.path,
                 "method": request.method,
             }
             logger.append_keys(fastapi=ctx)
             logger.info("Received request")
-            metrics.add_metric(
-                name="/".join(
-                    str(request.url.path).split("/")[:2]
-                ),  # enough detail to capture search IDs, but not individual tile coords
+
+            with single_metric(
+                name="RequestCount",
                 unit=MetricUnit.Count,
                 value=1,
-            )
+                default_dimensions=metrics.default_dimensions,
+                namespace="veda-backend",
+            ) as metric:
+                metric.add_dimension(
+                    name="route", value=f"{request.method} {self.path}"
+                )
+
             tracer.put_annotation(key="path", value=request.url.path)
             tracer.capture_method(original_route_handler)(request)
             return await original_route_handler(request)

raster_api/infrastructure/construct.py

@@ -80,6 +80,8 @@ def __init__(
             "VEDA_RASTER_ROOT_PATH", veda_raster_settings.raster_root_path
         )
 
+        veda_raster_function.add_environment("VEDA_RASTER_STAGE", stage)
+
         # Optional AWS S3 requester pays global setting
         if veda_raster_settings.raster_aws_request_payer:
             veda_raster_function.add_environment(

raster_api/runtime/src/config.py

@@ -57,6 +57,7 @@ class ApiSettings(BaseSettings):
     cachecontrol: str = "public, max-age=3600"
     debug: bool = False
     root_path: Optional[str] = None
+    stage: Optional[str] = None
 
     # MosaicTiler settings
     enable_mosaic_search: bool = False

raster_api/runtime/src/monitoring.py

@@ -1,42 +1,61 @@
 """Observability utils"""
+import json
 from typing import Callable
 
-from aws_lambda_powertools import Logger, Metrics, Tracer
+from aws_lambda_powertools import Logger, Metrics, Tracer, single_metric
 from aws_lambda_powertools.metrics import MetricUnit  # noqa: F401
+from src.config import ApiSettings
 
 from fastapi import Request, Response
 from fastapi.routing import APIRoute
 
+settings = ApiSettings()
+
 logger: Logger = Logger(service="raster-api", namespace="veda-backend")
-metrics: Metrics = Metrics(service="raster-api", namespace="veda-backend")
+metrics: Metrics = Metrics(namespace="veda-backend")
+metrics.set_default_dimensions(environment=settings.stage, service="raster-api")
 tracer: Tracer = Tracer()
 
 
 class LoggerRouteHandler(APIRoute):
-    """Extension of base APIRoute to add context to log statements, as well as record usage metricss"""
+    """Extension of base APIRoute to add context to log statements, as well as record usage metrics"""
 
     def get_route_handler(self) -> Callable:
         """Overide route handler method to add logs, metrics, tracing"""
         original_route_handler = super().get_route_handler()
 
         async def route_handler(request: Request) -> Response:
             # Add fastapi context to logs
+            body = await request.body()
+            try:
+                body_json = json.loads(body)
+            except json.decoder.JSONDecodeError:
+                body_json = None
+
             ctx = {
                 "path": request.url.path,
+                "path_params": request.path_params,
+                "body": body_json,
                 "route": self.path,
                 "method": request.method,
             }
             logger.append_keys(fastapi=ctx)
             logger.info("Received request")
-            metrics.add_metric(
-                name="/".join(
-                    str(request.url.path).split("/")[:2]
-                ),  # enough detail to capture search IDs, but not individual tile coords
+
+            with single_metric(
+                name="RequestCount",
                 unit=MetricUnit.Count,
                 value=1,
-            )
+                default_dimensions=metrics.default_dimensions,
+                namespace="veda-backend",
+            ) as metric:
+                metric.add_dimension(
+                    name="route", value=f"{request.method} {self.path}"
+                )
+
             tracer.put_annotation(key="path", value=request.url.path)
             tracer.capture_method(original_route_handler)(request)
+
             return await original_route_handler(request)
 
         return route_handler

stac_api/infrastructure/construct.py

@@ -84,6 +84,8 @@ def __init__(
             "VEDA_STAC_ROOT_PATH", veda_stac_settings.stac_root_path
         )
 
+        lambda_function.add_environment("VEDA_STAC_STAGE", stage)
+
         integration_kwargs = dict(handler=lambda_function)
         if veda_stac_settings.custom_host:
             integration_kwargs[

stac_api/runtime/src/app.py

@@ -8,7 +8,7 @@
 from src.config import post_request_model as POSTModel
 from src.extension import TiTilerExtension
 
-from fastapi import FastAPI
+from fastapi import APIRouter, FastAPI
 from fastapi.responses import ORJSONResponse
 from stac_fastapi.pgstac.db import close_db_connection, connect_to_db
 from starlette.middleware.cors import CORSMiddleware
@@ -19,7 +19,7 @@
 
 from .api import VedaStacApi
 from .core import VedaCrudClient
-from .monitoring import logger, metrics, tracer
+from .monitoring import LoggerRouteHandler, logger, metrics, tracer
 
 try:
     from importlib.resources import files as resources_files  # type: ignore
@@ -49,6 +49,7 @@
     search_post_request_model=POSTModel,
     response_class=ORJSONResponse,
     middlewares=[CompressionMiddleware],
+    router=APIRouter(route_class=LoggerRouteHandler),
 )
 app = api.app
 

stac_api/runtime/src/config.py

@@ -56,6 +56,7 @@ class _ApiSettings(pydantic.BaseSettings):
     debug: bool = False
     root_path: Optional[str] = None
     pgstac_secret_arn: Optional[str]
+    stage: Optional[str] = None
 
     @pydantic.validator("cors_origins")
     def parse_cors_origin(cls, v):

stac_api/runtime/src/monitoring.py

@@ -1,40 +1,58 @@
 """Observability utils"""
+import json
 from typing import Callable
 
-from aws_lambda_powertools import Logger, Metrics, Tracer
+from aws_lambda_powertools import Logger, Metrics, Tracer, single_metric
 from aws_lambda_powertools.metrics import MetricUnit  # noqa: F401
+from src.config import ApiSettings
 
 from fastapi import Request, Response
 from fastapi.routing import APIRoute
 
+settings = ApiSettings()
+
 logger: Logger = Logger(service="stac-api", namespace="veda-backend")
-metrics: Metrics = Metrics(service="stac-api", namespace="veda-backend")
+metrics: Metrics = Metrics(namespace="veda-backend")
+metrics.set_default_dimensions(environment=settings.stage, service="stac-api")
 tracer: Tracer = Tracer()
 
 
 class LoggerRouteHandler(APIRoute):
-    """Extension of base APIRoute to add context to log statements, as well as record usage metricss"""
+    """Extension of base APIRoute to add context to log statements, as well as record usage metrics"""
 
     def get_route_handler(self) -> Callable:
         """Overide route handler method to add logs, metrics, tracing"""
         original_route_handler = super().get_route_handler()
 
         async def route_handler(request: Request) -> Response:
             # Add fastapi context to logs
+            body = await request.body()
+            try:
+                body_json = json.loads(body)
+            except json.decoder.JSONDecodeError:
+                body_json = None
+
             ctx = {
                 "path": request.url.path,
+                "path_params": request.path_params,
+                "body": body_json,
                 "route": self.path,
                 "method": request.method,
             }
             logger.append_keys(fastapi=ctx)
             logger.info("Received request")
-            metrics.add_metric(
-                name="/".join(
-                    str(request.url).split("/")[:2]
-                ),  # enough detail to capture search IDs, but not individual tile coords
+
+            with single_metric(
+                name="RequestCount",
                 unit=MetricUnit.Count,
                 value=1,
-            )
+                default_dimensions=metrics.default_dimensions,
+                namespace="veda-backend",
+            ) as metric:
+                metric.add_dimension(
+                    name="route", value=f"{request.method} {self.path}"
+                )
+
             tracer.put_annotation(key="path", value=request.url.path)
             tracer.capture_method(original_route_handler)(request)
             return await original_route_handler(request)