Add tracing adjusting startup to prevent multiple FastAPI apps

.devcontainer/devcontainer.json

@@ -40,9 +40,7 @@
         "--security-opt=label=disable"
     ],
     // Remove this before pushing, only to allow local testing with editable library
-    "mounts": [
-        "source=/scratch/athena/observability-utils,target=/scratch/athena/observability-utils,type=bind,consistency=cached"
-    ],    
+    "mounts": [],
     // Mount the parent as /workspaces so we can pip install peers as editable
     "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind",
     // After the container is created, install the python project in editable form

.vscode/launch.json

@@ -25,25 +25,35 @@
             "request": "launch",
             "justMyCode": false,
             "module": "blueapi",
-            "args": [
-                "serve"
-            ]
+            "env": {
+                "OTLP_EXPORT_ENABLED": "false"
+            },
+            "args": "-c ${input:config_path} serve"
         },
         {
             "name": "Blueapi Controller",
             "type": "debugpy",
             "request": "launch",
             "justMyCode": false,
             "module": "blueapi",
-            "args": "controller ${input:args}"
+            "env": {
+                "OTLP_EXPORT_ENABLED": "false"
+            },
+            "args": "-c ${input:config_path} controller ${input:args}"
         },
     ],
     "inputs": [
+        {
+            "id": "config_path",
+            "type": "promptString",
+            "description": "Path to configuration YAML file",
+            "default": ""
+        },
         {
             "id": "args",
             "type": "promptString",
             "description": "Arguments to pass to controller",
             "default": ""
         }
     ]
-}
+}

Dockerfile

@@ -12,12 +12,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 RUN python -m venv /venv
 ENV PATH=/venv/bin:$PATH
 
+ENV OTLP_EXPORT_ENABLED=false
 # enable opentelemetry support
 ENV OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf
-# Chnage this to point to Jaeger server before merging
+# Change this to point to Jaeger server before merging e.g. https://daq-services-jaeger
 ENV OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318
-# Change this to enable proper secrity before merging
-ENV OTEL_EXPORTER_OTLP_INSECURE=true
 # Ensure that all Http headers are captured
 ENV OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST=".*"
 ENV OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE=".*"

dev-requirements.txt

@@ -9,8 +9,8 @@ annotated-types==0.7.0
 anyio==4.4.0
 appdirs==1.4.4
 asciitree==0.3.3
+asgiref==3.8.1
 asttokens==2.4.1
-async-timeout==4.0.3
 attrs==24.2.0
 babel==2.16.0
 beautifulsoup4==4.12.3
@@ -20,7 +20,6 @@ bluesky-kafka==0.10.0
 bluesky-live==0.0.8
 bluesky-stomp==0.1.2
 boltons==24.0.0
-bump-pydantic==0.8.0
 cachetools==5.5.0
 caproto==1.1.1
 certifi==2024.8.30
@@ -42,6 +41,7 @@ databroker==1.2.5
 dataclasses-json==0.6.7
 decorator==5.1.1
 deepmerge==2.0
+Deprecated==1.2.14
 distlib==0.3.8
 dls-bluesky-core==0.0.4
 dls-dodal==1.31.1
@@ -54,7 +54,6 @@ email_validator==2.2.0
 entrypoints==0.4
 epicscorelibs==7.0.7.99.0.2
 event-model==1.21.0
-exceptiongroup==1.2.2
 executing==2.1.0
 fastapi==0.114.2
 fastapi-cli==0.0.5
@@ -68,7 +67,9 @@ fsspec==2024.9.0
 funcy==2.0
 gitdb==4.0.11
 GitPython==3.1.43
+googleapis-common-protos==1.65.0
 graypy==2.1.0
+grpcio==1.66.2
 h11==0.14.0
 h5py==3.11.0
 HeapDict==1.0.1
@@ -81,7 +82,7 @@ identify==2.6.1
 idna==3.10
 imageio==2.35.1
 imagesize==1.4.1
-importlib_metadata==8.5.0
+importlib_metadata==8.4.0
 importlib_resources==6.4.5
 iniconfig==2.0.0
 intake==0.6.4
@@ -96,8 +97,6 @@ jsonschema-specifications==2023.12.1
 jupyterlab_widgets==3.0.13
 kiwisolver==1.4.7
 ldap3==2.9.1
-libcst==1.4.0
-livereload==2.7.0
 locket==1.0.0
 lz4==4.3.3
 markdown-it-py==3.0.0
@@ -122,7 +121,21 @@ nose2==0.15.1
 nslsii==0.10.3
 numcodecs==0.13.0
 numpy==1.26.4
+observability-utils==0.1.2
 opencv-python-headless==4.10.0.84
+opentelemetry-api==1.27.0
+opentelemetry-distro==0.48b0
+opentelemetry-exporter-otlp==1.27.0
+opentelemetry-exporter-otlp-proto-common==1.27.0
+opentelemetry-exporter-otlp-proto-grpc==1.27.0
+opentelemetry-exporter-otlp-proto-http==1.27.0
+opentelemetry-instrumentation==0.48b0
+opentelemetry-instrumentation-asgi==0.48b0
+opentelemetry-instrumentation-fastapi==0.48b0
+opentelemetry-proto==1.27.0
+opentelemetry-sdk==1.27.0
+opentelemetry-semantic-conventions==0.48b0
+opentelemetry-util-http==0.48b0
 ophyd==1.9.0
 ophyd-async==0.5.2
 orjson==3.10.7
@@ -147,6 +160,7 @@ ply==3.11
 pre-commit==3.8.0
 prettytable==3.11.0
 prompt-toolkit==3.0.36
+protobuf==4.25.5
 psutil==6.0.0
 ptyprocess==0.7.0
 pure_eval==0.2.3
@@ -173,7 +187,6 @@ python-dotenv==1.0.1
 python-multipart==0.0.9
 pytz==2024.2
 PyYAML==6.0.2
-pyyaml-include==2.1
 questionary==2.0.1
 redis==5.0.8
 redis-json-dict==0.2.0
@@ -217,9 +230,7 @@ suitcase-msgpack==0.3.0
 suitcase-utils==0.5.4
 super-state-machine==2.0.2
 tifffile==2024.8.30
-tomli==2.0.1
 toolz==0.12.1
-tornado==6.4.1
 tox==3.28.0
 tox-direct==0.4
 tqdm==4.66.5
@@ -244,6 +255,7 @@ websocket-client==1.8.0
 websockets==13.0.1
 widgetsnbextension==4.0.13
 workflows==2.27
+wrapt==1.16.0
 xarray==2024.9.0
 yarl==1.11.1
 zarr==2.18.3

docs/reference/openapi.yaml

@@ -179,6 +179,10 @@ components:
           default: true
           title: Is Pending
           type: boolean
+        request_id:
+          default: ''
+          title: Request Id
+          type: string
         task:
           title: Task
         task_id:

helm/blueapi/values.yaml

@@ -77,29 +77,29 @@ listener:
 
 # Additional envVars to mount to the pod as a String
 extraEnvVars: |
+  - name: OTLP_EXPORT_ENABLED
+    value: {{ .Values.tracing.otlp.export_enabled] }}
   - name: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL
-    value: {{ .Values.jaeger.otlp.protocol }}
+    value: {{ .Values.tracing.otlp.protocol }}
   - name: OTEL_EXPORTER_OTLP_ENDPOINT
-    value: "{{ .Values.jaeger.otlp.host }}:{{ .Values.jaeger.otlp.port }}"
-  - name: OTEL_EXPORTER_OTLP_INSECURE
-    value: "{{ .Values.jaeger.otlp.insecure }}"
+    value: "{{ .Values.tracing.otlp.host }}:{{ .Values.tracing.otlp.port }}"
   - name: OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST
-    value: {{ .Values.jaeger.otlp.request.headers }}
+    value: {{ .Values.tracing.http.request.headers }}
   - name: OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE
-    value: {{ .Values.jaeger.otlp.response.headers }}
+    value: {{ .Values.tracing.http.response.headers }}
 # - name: RABBITMQ_PASSWORD
 #   valueFrom:
 #     secretKeyRef:
 #       name: rabbitmq-password
 #       key: rabbitmq-password
 
-jaeger:
+tracing:
   otlp:
+    export_enabled: false
     protocol: http/protobuf
-    insecure: true
-    #
-    host: http://localhost
+    host: https://daq-services-jaeger # replace with central instance
     port: 4318
+  http:
     request:
       headers: ".*"
     response:
@@ -126,6 +126,8 @@ worker:
       password: guest
     host: rabbitmq
     port: 61613
+  tracing_exporter:
+    host: ""
 
 initContainer:
   scratch:

pyproject.toml

@@ -27,14 +27,14 @@ dependencies = [
     "fastapi>=0.112.0",
     "uvicorn",
     "requests",
-    "dls-bluesky-core",                                                  #requires ophyd-async
+    "dls-bluesky-core",                              #requires ophyd-async
     "dls-dodal>=1.31.0",
-    "super-state-machine",                                               # See GH issue 553
+    "super-state-machine",                           # See GH issue 553
     "GitPython",
     "bluesky-stomp>=0.1.2",
     "opentelemetry-distro>=0.48b0",
     "opentelemetry-instrumentation-fastapi>=0.48b0",
-    "observability-utils @ file:///scratch/athena/observability-utils/", # Uncomment this before merging
+    "observability-utils>=0.1.2",
 ]
 dynamic = ["version"]
 license.file = "LICENSE"
@@ -82,7 +82,7 @@ write_to = "src/blueapi/_version.py"
 
 [tool.mypy]
 ignore_missing_imports = true # Ignore missing stubs in imported modules
-namespace_packages = false    # rely only on __init__ files to determine fully qualified module names.
+namespace_packages = true     # necessary for tracing sdk to work with mypy
 
 [tool.pytest.ini_options]
 # Run pytest with all our checkers, and don't spam us with massive tracebacks on error

src/blueapi/cli/cli.py

@@ -10,6 +10,8 @@
 from bluesky_stomp.messaging import MessageContext, StompClient
 from bluesky_stomp.models import Broker
 from observability_utils.tracing import setup_tracing
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.trace import get_tracer_provider
 from pydantic import ValidationError
 from requests.exceptions import ConnectionError
 
@@ -19,14 +21,7 @@
 from blueapi.client.event_bus import AnyEvent, BlueskyStreamingError, EventBusClient
 from blueapi.client.rest import BlueskyRemoteControlError
 from blueapi.config import ApplicationConfig, ConfigLoader
-from blueapi.core import DataEvent
-from blueapi.service.main import start
-from blueapi.service.openapi import (
-    DOCS_SCHEMA_LOCATION,
-    generate_schema,
-    print_schema_as_yaml,
-    write_schema_as_yaml,
-)
+from blueapi.core import OTLP_EXPORT_ENABLED, DataEvent
 from blueapi.worker import ProgressEvent, Task, WorkerEvent
 
 from .scratch import setup_scratch
@@ -42,7 +37,6 @@
 def main(ctx: click.Context, config: Path | None | tuple[Path, ...]) -> None:
     # if no command is supplied, run with the options passed
 
-    setup_tracing("BlueAPI")  # initialise TracerProvider for server app
     config_loader = ConfigLoader(ApplicationConfig)
     if config is not None:
         configs = (config,) if isinstance(config, Path) else config
@@ -72,6 +66,16 @@ def main(ctx: click.Context, config: Path | None | tuple[Path, ...]) -> None:
     help="[Development only] update the schema in the documentation",
 )
 def schema(output: Path | None = None, update: bool = False) -> None:
+    """Only import the service functions when starting the service or generating
+    the schema, not the controller as a new FastAPI app will be started each time.
+    """
+    from blueapi.service.openapi import (
+        DOCS_SCHEMA_LOCATION,
+        generate_schema,
+        print_schema_as_yaml,
+        write_schema_as_yaml,
+    )
+
     """Generate the schema for the REST API"""
     schema = generate_schema()
 
@@ -89,6 +93,17 @@ def start_application(obj: dict):
     """Run a worker that accepts plans to run"""
     config: ApplicationConfig = obj["config"]
 
+    """Only import the service functions when starting the service or generating
+    the schema, not the controller as a new FastAPI app will be started each time.
+    """
+    from blueapi.service.main import app, start
+
+    """
+    Set up basic automated instrumentation for the FastAPI app, creating the
+    observability context.
+    """
+    setup_tracing("BlueAPI", OTLP_EXPORT_ENABLED)
+    FastAPIInstrumentor().instrument_app(app, tracer_provider=get_tracer_provider())
     start(config)
 
 
@@ -103,7 +118,7 @@ def start_application(obj: dict):
 def controller(ctx: click.Context, output: str) -> None:
     """Client utility for controlling and introspecting the worker"""
 
-    setup_tracing("BlueAPICLI")  # initialise TracerProvider for controller app
+    setup_tracing("BlueAPICLI", OTLP_EXPORT_ENABLED)
     if ctx.invoked_subcommand is None:
         print("Please invoke subcommand!")
         return

src/blueapi/client/rest.py

@@ -135,11 +135,12 @@ def _request_and_deserialize(
         get_exception: Callable[[requests.Response], Exception | None] = _exception,
     ) -> T:
         url = self._url(suffix)
+        # Get the trace context to propagate to the REST API
         carr = get_context_propagator()
         if data:
             response = requests.request(method, url, json=data, headers=carr)
         else:
-            response = requests.request(method, url)
+            response = requests.request(method, url, headers=carr)
         exception = get_exception(response)
         if exception is not None:
             raise exception

src/blueapi/config.py

@@ -23,24 +23,6 @@ class Source(BaseModel):
     module: Path | str
 
 
-# class BasicAuthentication(BaseModel):
-#     """
-#     Log in details for when a server uses authentication.
-#     If username or passcode match exactly the regex ^\\${(.*)}$
-#     they attempt to replace with an environment variable of the same.
-#     i.e. ${foo} or ${FOO} are replaced with the value of FOO
-#     """
-
-#     username: str = "test"  # "guest"
-#     passcode: str = "test"  # "guest"
-
-#     @validator("username", "passcode")
-#     def get_from_env(cls, v: str):
-#         if v.startswith("${") and v.endswith("}"):
-#             return os.environ[v.removeprefix("${").removesuffix("}").upper()]
-#         return v
-
-
 class StompConfig(BaseModel):
     """
     Config for connecting to stomp broker