Integrate the Armory app with Open Telemetry

apps/armory/.env.default

@@ -12,7 +12,8 @@ ADMIN_API_KEY=171c50ec62122b8c08362dcf9dce9b016ed615cfc7b90d4bc3fe5b223d967fb2
 # APP db connection string
 APP_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/armory?schema=public
 
-# MIGRATOR db credentials. host/port/name should be the same, username&password may be different
+# Migrator db credentials.
+# The host/port/name should be the same, username&password may be different
 APP_DATABASE_USERNAME=postgres
 APP_DATABASE_PASSWORD=postgres
 APP_DATABASE_HOST=host.docker.internal
@@ -30,4 +31,11 @@ HISTORICAL_TRANSFER_FEED_PRIVATE_KEY=0xf5c8f17cc09215c5038f6b8d5e557c0d98d341236
 POLICY_ENGINE_URLS=http://localhost:3010
 POLICY_ENGINE_ADMIN_API_KEYS=engine-admin-api-key
 
-MANAGED_DATASTORE_BASE_URL=http://localhost:3005/v1/data
+MANAGED_DATASTORE_BASE_URL=http://localhost:3005/v1/data
+
+# OpenTelemetry configuration.
+#
+# NOTE: To disable OTEL, both OTEL_METRIC_EXPORTER_URL and
+# OTEL_TRACE_EXPORTER_URL must not be set.
+OTEL_METRIC_EXPORTER_URL=http://localhost:4318/v1/metrics
+OTEL_TRACE_EXPORTER_URL=http://localhost:4318/v1/traces

apps/armory/src/armory.config.ts

@@ -44,7 +44,11 @@ const configSchema = z.object({
     priceFeedPrivateKey: hexSchema,
     historicalTransferFeedPrivateKey: hexSchema
   }),
-  managedDataStoreBaseUrl: z.string().url()
+  managedDataStoreBaseUrl: z.string().url(),
+  openTelemetry: z.object({
+    metricExporterUrl: z.string().url().optional(),
+    traceExporterUrl: z.string().url().optional()
+  })
 })
 
 export type Config = z.infer<typeof configSchema>
@@ -81,7 +85,11 @@ export const load = (): Config => {
       priceFeedPrivateKey: process.env.PRICE_FEED_PRIVATE_KEY,
       historicalTransferFeedPrivateKey: process.env.HISTORICAL_TRANSFER_FEED_PRIVATE_KEY
     },
-    managedDataStoreBaseUrl: process.env.MANAGED_DATASTORE_BASE_URL
+    managedDataStoreBaseUrl: process.env.MANAGED_DATASTORE_BASE_URL,
+    openTelemetry: {
+      metricExporterUrl: process.env.OTEL_METRIC_EXPORTER_URL,
+      traceExporterUrl: process.env.OTEL_TRACE_EXPORTER_URL
+    }
   })
 
   if (result.success) {

apps/armory/src/armory.module.ts

@@ -1,5 +1,5 @@
-import { ConfigModule } from '@narval/config-module'
-import { HttpLoggerMiddleware, LoggerModule } from '@narval/nestjs-shared'
+import { ConfigModule, ConfigService } from '@narval/config-module'
+import { HttpLoggerMiddleware, LoggerModule, LoggerService, OpenTelemetryModule } from '@narval/nestjs-shared'
 import { ClassSerializerInterceptor, MiddlewareConsumer, Module, NestModule } from '@nestjs/common'
 import { APP_INTERCEPTOR } from '@nestjs/core'
 import { AppModule } from './app/app.module'
@@ -9,6 +9,7 @@ import { ArmoryController } from './armory.controller'
 import { ClientModule } from './client/client.module'
 import { ManagedDataStoreModule } from './managed-data-store/managed-data-store.module'
 import { OrchestrationModule } from './orchestration/orchestration.module'
+import { OpenTelemetryModuleOptionFactory } from './shared/factory/open-telemetry-module-option.factory'
 import { QueueModule } from './shared/module/queue/queue.module'
 import { TransferTrackingModule } from './transfer-tracking/transfer-tracking.module'
 
@@ -18,7 +19,11 @@ const INFRASTRUCTURE_MODULES = [
     load: [load],
     isGlobal: true
   }),
-  QueueModule.forRoot()
+  QueueModule.forRoot(),
+  OpenTelemetryModule.registerAsync({
+    inject: [LoggerService, ConfigService],
+    useClass: OpenTelemetryModuleOptionFactory
+  })
 ]
 
 const DOMAIN_MODULES = [AppModule, OrchestrationModule, TransferTrackingModule, ManagedDataStoreModule, ClientModule]

apps/armory/src/managed-data-store/http/rest/controller/data-store.controller.ts

@@ -1,3 +1,4 @@
+import { MetricService, TraceService } from '@narval/nestjs-shared'
 import { Criterion, EntityUtil, Then, UserRole } from '@narval/policy-engine-shared'
 import { Body, Controller, Get, HttpCode, HttpStatus, Post, Query, UseGuards } from '@nestjs/common'
 import { ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger'
@@ -22,7 +23,9 @@ export class DataStoreController {
   constructor(
     private entityDataStoreService: EntityDataStoreService,
     private policyDataStoreService: PolicyDataStoreService,
-    private clusterService: ClusterService
+    private clusterService: ClusterService,
+    private traceService: TraceService,
+    private metricService: MetricService
   ) {}
 
   @Get('/entities')
@@ -35,18 +38,24 @@ export class DataStoreController {
     type: EntityDataStoreDto
   })
   async getEntities(@Query('clientId') clientId: string): Promise<EntityDataStoreDto> {
-    const entity = await this.entityDataStoreService.getEntities(clientId)
+    this.metricService.createCounter('data_store_entity_get_count').add(1, { clientId })
 
-    if (entity) {
-      return { entity }
-    }
+    return this.traceService.startActiveSpan('dataStore.entity.get', async (span) => {
+      span.setAttributes({ clientId })
 
-    return {
-      entity: {
-        data: EntityUtil.empty(),
-        signature: ''
+      const entity = await this.entityDataStoreService.getEntities(clientId)
+
+      if (entity) {
+        return { entity }
+      }
+
+      return {
+        entity: {
+          data: EntityUtil.empty(),
+          signature: ''
+        }
       }
-    }
+    })
   }
 
   @Get('/policies')
@@ -59,30 +68,36 @@ export class DataStoreController {
     type: PolicyDataStoreDto
   })
   async getPolicies(@Query('clientId') clientId: string): Promise<PolicyDataStoreDto> {
-    const policy = await this.policyDataStoreService.getPolicies(clientId)
-
-    if (policy) {
-      return { policy }
-    }
-
-    return {
-      policy: {
-        data: [
-          {
-            id: 'admins-full-access',
-            description: 'Admins get full access',
-            when: [
-              {
-                criterion: Criterion.CHECK_PRINCIPAL_ROLE,
-                args: [UserRole.ADMIN]
-              }
-            ],
-            then: Then.PERMIT
-          }
-        ],
-        signature: ''
+    this.metricService.createCounter('data_store_policy_get_count').add(1, { clientId })
+
+    return this.traceService.startActiveSpan('dataStore.policy.get', async (span) => {
+      span.setAttributes({ clientId })
+
+      const policy = await this.policyDataStoreService.getPolicies(clientId)
+
+      if (policy) {
+        return { policy }
       }
-    }
+
+      return {
+        policy: {
+          data: [
+            {
+              id: 'admins-full-access',
+              description: 'Admins get full access',
+              when: [
+                {
+                  criterion: Criterion.CHECK_PRINCIPAL_ROLE,
+                  args: [UserRole.ADMIN]
+                }
+              ],
+              then: Then.PERMIT
+            }
+          ],
+          signature: ''
+        }
+      }
+    })
   }
 
   @Post('/entities')
@@ -94,7 +109,13 @@ export class DataStoreController {
     type: SetEntityStoreResponseDto
   })
   setEntities(@Query('clientId') clientId: string, @Body() body: SetEntityStoreDto) {
-    return this.entityDataStoreService.setEntities(clientId, body)
+    this.metricService.createCounter('data_store_entity_set_count').add(1, { clientId })
+
+    return this.traceService.startActiveSpan('dataStore.entity.set', (span) => {
+      span.setAttributes({ clientId })
+
+      return this.entityDataStoreService.setEntities(clientId, body)
+    })
   }
 
   @Post('/policies')
@@ -109,7 +130,13 @@ export class DataStoreController {
     @Query('clientId') clientId: string,
     @Body() body: SetPolicyStoreDto
   ): Promise<SetPolicyStoreResponseDto> {
-    return this.policyDataStoreService.setPolicies(clientId, body)
+    this.metricService.createCounter('data_store_policy_set_count').add(1, { clientId })
+
+    return this.traceService.startActiveSpan('dataStore.policy.set', (span) => {
+      span.setAttributes({ clientId })
+
+      return this.policyDataStoreService.setPolicies(clientId, body)
+    })
   }
 
   @Post('/sync')
@@ -123,20 +150,26 @@ export class DataStoreController {
     type: SyncDto
   })
   async sync(@ClientId('clientId') clientId: string): Promise<SyncDto> {
-    try {
-      const success = await this.clusterService.sync(clientId)
+    this.metricService.createCounter('data_store_cluster_sync_count').add(1, { clientId })
 
-      return {
-        latestSync: {
-          success
+    return this.traceService.startActiveSpan('dataStore.cluster.sync', async (span) => {
+      span.setAttributes({ clientId })
+
+      try {
+        const success = await this.clusterService.sync(clientId)
+
+        return {
+          latestSync: {
+            success
+          }
         }
+      } catch (error) {
+        return SyncDto.create({
+          latestSync: {
+            success: false
+          }
+        })
       }
-    } catch (error) {
-      return SyncDto.create({
-        latestSync: {
-          success: false
-        }
-      })
-    }
+    })
   }
 }

apps/armory/src/policy-engine/http/client/policy-engine.client.ts

@@ -1,4 +1,4 @@
-import { LoggerService } from '@narval/nestjs-shared'
+import { LoggerService, TraceService } from '@narval/nestjs-shared'
 import {
   CreateClient,
   EvaluationRequest,
@@ -17,7 +17,8 @@ export class PolicyEngineClientException extends ApplicationException {}
 export class PolicyEngineClient {
   constructor(
     private httpService: HttpService,
-    private logger: LoggerService
+    private logger: LoggerService,
+    private traceService: TraceService
   ) {}
 
   async evaluate(option: {
@@ -135,9 +136,15 @@ export class PolicyEngineClient {
   }
 
   private getHeaders(option: { clientId?: string; clientSecret?: string }) {
+    const activeSpan = this.traceService.getActiveSpan()
+
     return {
       'x-client-id': option.clientId,
-      'x-client-secret': option.clientSecret
+      'x-client-secret': option.clientSecret,
+      // Trace context headers.
+      // See https://www.w3.org/TR/trace-context/
+      traceparent: activeSpan?.spanContext().traceId,
+      tracestate: activeSpan?.spanContext().spanId
     }
   }
 }

docker-compose.yml

@@ -30,6 +30,27 @@ services:
     volumes:
       - ./.data/redis:/data
 
+  jaeger:
+    image: 'jaegertracing/all-in-one'
+    ports:
+      - '16686:16686' # Web UI
+      - '14250:14250' # gRPC
+    networks:
+      - armory_stack
+
+  otel_collector:
+    image: 'otel/opentelemetry-collector-contrib'
+    command: ['--config=/etc/otel-collector-config.yaml']
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - '4318:4318' # HTTP
+      - '4317:4317' # gRPC
+    networks:
+      - armory_stack
+    depends_on:
+      - jaeger
+
   armory:
     image: armory/local:latest
     build:

otel-collector-config.yaml

@@ -0,0 +1,31 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+exporters:
+  debug:
+    verbosity: detailed
+  otlp:
+    endpoint: jaeger:4317
+    tls:
+      insecure: true
+    sending_queue:
+      enabled: true
+    retry_on_failure:
+      enabled: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [debug, otlp]
+    metrics:
+      receivers: [otlp]
+      exporters: [debug]
+    logs:
+      receivers: [otlp]
+      exporters: [debug]