Dynamically load api config and refactor the logger (#51)

* WIP - add cloudformation.yaml (health checks failing)

* WIP - load dynamic config

* Extract config get/set functions

* Extract logger options to env variables

* Make logger usage consistent in data-pusher

* Moving more logger options and fixing tests

* Load env in data-pusher

* Remove cloudformation.yaml for now

* Fix TS configuration for building project

* Improve validation for ENVs

---------

Co-authored-by: Emanuel Tesař <e.tesarr@gmail.com>

.github/workflows/main.yml

@@ -36,6 +36,8 @@ jobs:
         run: pnpm install --frozen-lockfile
       - name: Build
         run: pnpm run build
+      - name: Lint Typescript
+        run: pnpm run tsc
       - name: Lint
         run: pnpm run prettier:check && pnpm run eslint:check
       - name: Test

package.json

@@ -12,7 +12,8 @@
     "eslint:fix": "pnpm recursive run eslint:fix",
     "prettier:check": "pnpm recursive run prettier:check",
     "prettier:fix": "pnpm recursive run prettier:fix",
-    "test": "pnpm recursive run test"
+    "test": "pnpm recursive run test",
+    "tsc": "pnpm recursive run tsc"
   },
   "keywords": [],
   "license": "MIT",

packages/api/.env.example

@@ -0,0 +1,16 @@
+LOGGER_ENABLED=true
+LOG_COLORIZE=true
+LOG_FORMAT=pretty
+LOG_LEVEL=info
+
+# Available options
+#   local (default) - loads config/signed-api.json from the filesystem
+#   aws-s3 - loads the config file from AWS S3
+CONFIG_SOURCE=local
+
+# Set these variables if you would like to source your config from AWS S3
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_REGION=
+AWS_S3_BUCKET_NAME=
+AWS_S3_BUCKET_PATH=

packages/api/README.md

@@ -6,7 +6,8 @@ A service for storing and accessing signed data. It provides endpoints to handle
 
 1. `cp config/signed-api.example.json config/signed-api.json` - To create a config file from the example one. Optionally
    change the defaults.
-2. `pnpm run dev` - To start the API server. The port number can be configured in the configuration file.
+2. `cp .env.example .env` - To copy the example environment variables. Optionally change the defaults.
+3. `pnpm run dev` - To start the API server. The port number can be configured in the configuration file.
 
 ## Deployment
 

packages/api/config/signed-api.example.json

@@ -13,10 +13,5 @@
   "port": 8090,
   "cache": {
     "maxAgeSeconds": 300
-  },
-  "logger": {
-    "type": "pretty",
-    "styling": "on",
-    "minLevel": "debug"
   }
 }

packages/api/package.json

@@ -7,7 +7,7 @@
   },
   "main": "index.js",
   "scripts": {
-    "build": "tsc --project .",
+    "build": "tsc --project tsconfig.build.json",
     "clean": "rm -rf coverage dist",
     "dev": "nodemon --ext ts,js,json,env  --exec \"pnpm ts-node src/index.ts\"",
     "docker:build": "docker compose --file docker/docker-compose.yml build",
@@ -19,7 +19,8 @@
     "prettier:check": "prettier --check \"./**/*.{js,ts,md,json}\"",
     "prettier:fix": "prettier --write \"./**/*.{js,ts,md,json}\"",
     "start-prod": "node dist/src/index.js",
-    "test": "jest"
+    "test": "jest",
+    "tsc": "tsc --project ."
   },
   "license": "MIT",
   "devDependencies": {
@@ -30,6 +31,8 @@
   },
   "dependencies": {
     "@api3/promise-utils": "0.4.0",
+    "@aws-sdk/client-s3": "^3.418.0",
+    "dotenv": "^16.3.1",
     "ethers": "^5.7.2",
     "express": "^4.18.2",
     "lodash": "^4.17.21",

packages/api/src/config.ts

@@ -0,0 +1,54 @@
+import { readFileSync } from 'fs';
+import { join } from 'path';
+import { go } from '@api3/promise-utils';
+import { S3 } from '@aws-sdk/client-s3';
+import { logger } from './logger';
+import { Config, configSchema } from './schema';
+import { loadEnv } from './env';
+
+let config: Config | undefined;
+
+export const getConfig = (): Config => {
+  if (!config) throw new Error(`Config has not been set yet`);
+
+  return config;
+};
+
+export const fetchAndCacheConfig = async (): Promise<Config> => {
+  const jsonConfig = await fetchConfig();
+  config = configSchema.parse(jsonConfig);
+  return config;
+};
+
+const fetchConfig = async (): Promise<any> => {
+  const env = loadEnv();
+  const source = env.CONFIG_SOURCE;
+  if (!source || source === 'local') {
+    return JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8'));
+  }
+  if (source === 'aws-s3') {
+    return await fetchConfigFromS3();
+  }
+  throw new Error(`Unable to load config CONFIG_SOURCE:${source}`);
+};
+
+const fetchConfigFromS3 = async (): Promise<any> => {
+  const env = loadEnv();
+  const region = env.AWS_REGION!; // Validated by environment variables schema.
+  const s3 = new S3({ region });
+
+  const params = {
+    Bucket: env.AWS_S3_BUCKET_NAME,
+    Key: env.AWS_S3_BUCKET_PATH,
+  };
+
+  logger.info(`Fetching config from AWS S3 region:${region}...`);
+  const res = await go(() => s3.getObject(params), { retries: 1 });
+  if (!res.success) {
+    logger.error('Error fetching config from AWS S3:', res.error);
+    throw res.error;
+  }
+  logger.info('Config fetched successfully from AWS S3');
+  const stringifiedConfig = await res.data.Body!.transformToString();
+  return JSON.parse(stringifiedConfig);
+};

packages/api/src/env.ts

@@ -0,0 +1,19 @@
+import { join } from 'path';
+import dotenv from 'dotenv';
+import { EnvConfig, envConfigSchema } from './schema';
+
+let env: EnvConfig | undefined;
+
+export const loadEnv = () => {
+  if (env) return env;
+
+  dotenv.config({ path: join(__dirname, '../.env') });
+
+  const parseResult = envConfigSchema.safeParse(process.env);
+  if (!parseResult.success) {
+    throw new Error(`Invalid environment variables:\n, ${JSON.stringify(parseResult.error.format())}`);
+  }
+
+  env = parseResult.data;
+  return env;
+};

packages/api/src/handlers.test.ts

@@ -2,7 +2,7 @@ import { readFileSync } from 'fs';
 import { join } from 'path';
 import { omit } from 'lodash';
 import * as cacheModule from './cache';
-import * as utilsModule from './utils';
+import * as configModule from './config';
 import { batchInsertData, getData, listAirnodeAddresses } from './handlers';
 import { createSignedData, generateRandomWallet } from '../test/utils';
 
@@ -12,7 +12,7 @@ afterEach(() => {
 
 beforeEach(() => {
   jest
-    .spyOn(utilsModule, 'getConfig')
+    .spyOn(configModule, 'getConfig')
     .mockImplementation(() => JSON.parse(readFileSync(join(__dirname, '../config/signed-api.example.json'), 'utf8')));
 });
 

packages/api/src/handlers.ts

@@ -4,8 +4,9 @@ import { CACHE_HEADERS, COMMON_HEADERS } from './constants';
 import { deriveBeaconId, recoverSignerAddress } from './evm';
 import { getAll, getAllAirnodeAddresses, prune, putAll } from './in-memory-cache';
 import { ApiResponse } from './types';
-import { generateErrorResponse, getConfig, isBatchUnique } from './utils';
+import { generateErrorResponse, isBatchUnique } from './utils';
 import { batchSignedDataSchema, evmAddressSchema } from './schema';
+import { getConfig } from './config';
 
 // Accepts a batch of signed data that is first validated for consistency and data integrity errors. If there is any
 // issue during this step, the whole batch is rejected.

packages/api/src/index.ts

@@ -5,8 +5,14 @@
 // create a test script with/without the source map support, build the project and run the built script using node.
 import 'source-map-support/register';
 import { startServer } from './server';
-import { getConfig } from './utils';
 import { logger } from './logger';
+import { fetchAndCacheConfig } from './config';
 
-logger.info('Using configuration', getConfig());
-startServer();
+async function main() {
+  const config = await fetchAndCacheConfig();
+  logger.info('Using configuration', config);
+
+  startServer(config);
+}
+
+main();

packages/api/src/logger.ts

@@ -1,5 +1,15 @@
-import { createLogger } from 'signed-api/common';
-import { getConfig } from './utils';
+import { createLogger, logConfigSchema } from 'signed-api/common';
+import { loadEnv } from './env';
 
-const config = getConfig();
-export const logger = createLogger(config.logger);
+// We need to load the environment variables before we can use the logger. Because we want the logger to always be
+// available, we load the environment variables as a side effect during the module import.
+const env = loadEnv();
+
+const options = logConfigSchema.parse({
+  colorize: env.LOG_COLORIZE,
+  enabled: env.LOGGER_ENABLED,
+  minLevel: env.LOG_LEVEL,
+  format: env.LOG_FORMAT,
+});
+
+export const logger = createLogger(options);

packages/api/src/schema.test.ts

@@ -1,7 +1,8 @@
 import { readFileSync } from 'fs';
 import { join } from 'path';
 import { ZodError } from 'zod';
-import { configSchema, endpointSchema, endpointsSchema } from './schema';
+import dotenv from 'dotenv';
+import { configSchema, endpointSchema, endpointsSchema, envBooleanSchema, envConfigSchema } from './schema';
 
 describe('endpointSchema', () => {
   it('validates urlPath', () => {
@@ -48,3 +49,66 @@ describe('configSchema', () => {
     expect(() => configSchema.parse(config)).not.toThrow();
   });
 });
+
+describe('env config schema', () => {
+  it('parses boolean env variable correctly', () => {
+    expect(envBooleanSchema.parse('true')).toBe(true);
+    expect(envBooleanSchema.parse('false')).toBe(false);
+
+    // Using a function to create the expected error because the error message length is too long to be inlined. The
+    // error messages is trivially stringified if propagated to the user.
+    const createExpectedError = (received: string) =>
+      new ZodError([
+        {
+          code: 'invalid_union',
+          unionErrors: [
+            new ZodError([
+              {
+                received,
+                code: 'invalid_literal',
+                expected: 'true',
+                path: [],
+                message: 'Invalid literal value, expected "true"',
+              },
+            ]),
+            new ZodError([
+              {
+                received,
+                code: 'invalid_literal',
+                expected: 'false',
+                path: [],
+                message: 'Invalid literal value, expected "false"',
+              },
+            ]),
+          ],
+          path: [],
+          message: 'Invalid input',
+        },
+      ]);
+    expect(() => envBooleanSchema.parse('')).toThrow(createExpectedError(''));
+    expect(() => envBooleanSchema.parse('off')).toThrow(createExpectedError('off'));
+  });
+
+  it('parses example env correctly', () => {
+    // Load the example configuration from the ".env.example" file
+    const env = dotenv.parse(readFileSync(join(__dirname, '../.env.example'), 'utf8'));
+
+    expect(() => envConfigSchema.parse(env)).not.toThrow();
+  });
+
+  it('AWS_REGION is set when CONFIG_SOURCE is aws-s3', () => {
+    const env = {
+      CONFIG_SOURCE: 'aws-s3',
+    };
+
+    expect(() => envConfigSchema.parse(env)).toThrow(
+      new ZodError([
+        {
+          code: 'custom',
+          message: 'The AWS_REGION must be set when CONFIG_SOURCE is "aws-s3"',
+          path: ['AWS_REGION'],
+        },
+      ])
+    );
+  });
+});

packages/api/src/schema.ts

@@ -1,5 +1,5 @@
 import { uniqBy } from 'lodash';
-import { logConfigSchema } from 'signed-api/common';
+import { logFormatSchema, logLevelSchema } from 'signed-api/common';
 import { z } from 'zod';
 
 export const endpointSchema = z
@@ -28,7 +28,6 @@ export const configSchema = z
     cache: z.object({
       maxAgeSeconds: z.number().nonnegative().int(),
     }),
-    logger: logConfigSchema,
   })
   .strict();
 
@@ -52,3 +51,35 @@ export type SignedData = z.infer<typeof signedDataSchema>;
 export const batchSignedDataSchema = z.array(signedDataSchema);
 
 export type BatchSignedData = z.infer<typeof batchSignedDataSchema>;
+
+export const envBooleanSchema = z.union([z.literal('true'), z.literal('false')]).transform((val) => val === 'true');
+
+// We apply default values to make it convenient to omit certain environment variables. The default values should be
+// primarily focused on users and production usage.
+export const envConfigSchema = z
+  .object({
+    LOGGER_ENABLED: envBooleanSchema.default('true'),
+    LOG_COLORIZE: envBooleanSchema.default('false'),
+    LOG_FORMAT: logFormatSchema.default('json'),
+    LOG_LEVEL: logLevelSchema.default('info'),
+
+    CONFIG_SOURCE: z.union([z.literal('local'), z.literal('aws-s3')]).default('local'),
+
+    AWS_ACCESS_KEY_ID: z.string().optional(),
+    AWS_SECRET_ACCESS_KEY: z.string().optional(),
+    AWS_REGION: z.string().optional(),
+    AWS_S3_BUCKET_NAME: z.string().optional(),
+    AWS_S3_BUCKET_PATH: z.string().optional(),
+  })
+  .strip() // We parse from ENV variables of the process which has many variables that we don't care about.
+  .superRefine((val, ctx) => {
+    if (val.CONFIG_SOURCE === 'aws-s3' && !val.AWS_REGION) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: 'The AWS_REGION must be set when CONFIG_SOURCE is "aws-s3"',
+        path: ['AWS_REGION'],
+      });
+    }
+  });
+
+export type EnvConfig = z.infer<typeof envConfigSchema>;

packages/api/src/server.ts

@@ -1,10 +1,9 @@
 import express from 'express';
 import { getData, listAirnodeAddresses, batchInsertData } from './handlers';
-import { getConfig } from './utils';
 import { logger } from './logger';
+import { Config } from './schema';
 
-export const startServer = () => {
-  const config = getConfig();
+export const startServer = (config: Config) => {
   const app = express();
 
   app.use(express.json());

packages/api/src/utils.ts

@@ -1,11 +1,13 @@
-import { readFileSync } from 'fs';
-import { join } from 'path';
-import { ApiResponse } from './types';
 import { COMMON_HEADERS } from './constants';
-import { BatchSignedData, SignedData, configSchema } from './schema';
+import { BatchSignedData, SignedData } from './schema';
+import { ApiResponse } from './types';
 
-export const isBatchUnique = (batchSignedData: BatchSignedData) =>
-  batchSignedData.length === new Set(batchSignedData.map(({ airnode, templateId }) => airnode.concat(templateId))).size;
+export const isBatchUnique = (batchSignedData: BatchSignedData) => {
+  return (
+    batchSignedData.length ===
+    new Set(batchSignedData.map(({ airnode, templateId }) => airnode.concat(templateId))).size
+  );
+};
 
 export const isIgnored = (signedData: SignedData, ignoreAfterTimestamp: number) => {
   return parseInt(signedData.timestamp) > ignoreAfterTimestamp;
@@ -19,6 +21,3 @@ export const generateErrorResponse = (
 ): ApiResponse => {
   return { statusCode, headers: COMMON_HEADERS, body: JSON.stringify({ message, detail, extra }) };
 };
-
-export const getConfig = () =>
-  configSchema.parse(JSON.parse(readFileSync(join(__dirname, '../config/signed-api.example.json'), 'utf8')));