Improve validation for ENVs

packages/api/.env.example

@@ -13,5 +13,4 @@ AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 AWS_REGION=
 AWS_S3_BUCKET_NAME=
-AWS_S3_BUCKET_PATH="path/to/config/inside/bucket/signed-api.json"
-
+AWS_S3_BUCKET_PATH=

packages/api/README.md

@@ -6,7 +6,8 @@ A service for storing and accessing signed data. It provides endpoints to handle
 
 1. `cp config/signed-api.example.json config/signed-api.json` - To create a config file from the example one. Optionally
    change the defaults.
-2. `pnpm run dev` - To start the API server. The port number can be configured in the configuration file.
+2. `cp .env.example .env` - To copy the example environment variables. Optionally change the defaults.
+3. `pnpm run dev` - To start the API server. The port number can be configured in the configuration file.
 
 ## Deployment
 

packages/api/src/config.ts

@@ -0,0 +1,54 @@
+import { readFileSync } from 'fs';
+import { join } from 'path';
+import { go } from '@api3/promise-utils';
+import { S3 } from '@aws-sdk/client-s3';
+import { logger } from './logger';
+import { Config, configSchema } from './schema';
+import { loadEnv } from './env';
+
+let config: Config | undefined;
+
+export const getConfig = (): Config => {
+  if (!config) throw new Error(`Config has not been set yet`);
+
+  return config;
+};
+
+export const fetchAndCacheConfig = async (): Promise<Config> => {
+  const jsonConfig = await fetchConfig();
+  config = configSchema.parse(jsonConfig);
+  return config;
+};
+
+const fetchConfig = async (): Promise<any> => {
+  const env = loadEnv();
+  const source = env.CONFIG_SOURCE;
+  if (!source || source === 'local') {
+    return JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8'));
+  }
+  if (source === 'aws-s3') {
+    return await fetchConfigFromS3();
+  }
+  throw new Error(`Unable to load config CONFIG_SOURCE:${source}`);
+};
+
+const fetchConfigFromS3 = async (): Promise<any> => {
+  const env = loadEnv();
+  const region = env.AWS_REGION!; // Validated by environment variables schema.
+  const s3 = new S3({ region });
+
+  const params = {
+    Bucket: env.AWS_S3_BUCKET_NAME,
+    Key: env.AWS_S3_BUCKET_PATH,
+  };
+
+  logger.info(`Fetching config from AWS S3 region:${region}...`);
+  const res = await go(() => s3.getObject(params), { retries: 1 });
+  if (!res.success) {
+    logger.error('Error fetching config from AWS S3:', res.error);
+    throw res.error;
+  }
+  logger.info('Config fetched successfully from AWS S3');
+  const stringifiedConfig = await res.data.Body!.transformToString();
+  return JSON.parse(stringifiedConfig);
+};

packages/api/src/env.ts

@@ -0,0 +1,19 @@
+import { join } from 'path';
+import dotenv from 'dotenv';
+import { EnvConfig, envConfigSchema } from './schema';
+
+let env: EnvConfig | undefined;
+
+export const loadEnv = () => {
+  if (env) return env;
+
+  dotenv.config({ path: join(__dirname, '../.env') });
+
+  const parseResult = envConfigSchema.safeParse(process.env);
+  if (!parseResult.success) {
+    throw new Error(`Invalid environment variables:\n, ${JSON.stringify(parseResult.error.format())}`);
+  }
+
+  env = parseResult.data;
+  return env;
+};

packages/api/src/handlers.test.ts

@@ -2,7 +2,7 @@ import { readFileSync } from 'fs';
 import { join } from 'path';
 import { omit } from 'lodash';
 import * as cacheModule from './cache';
-import * as utilsModule from './utils';
+import * as configModule from './config';
 import { batchInsertData, getData, listAirnodeAddresses } from './handlers';
 import { createSignedData, generateRandomWallet } from '../test/utils';
 
@@ -12,7 +12,7 @@ afterEach(() => {
 
 beforeEach(() => {
   jest
-    .spyOn(utilsModule, 'getConfig')
+    .spyOn(configModule, 'getConfig')
     .mockImplementation(() => JSON.parse(readFileSync(join(__dirname, '../config/signed-api.example.json'), 'utf8')));
 });
 

packages/api/src/handlers.ts

@@ -4,8 +4,9 @@ import { CACHE_HEADERS, COMMON_HEADERS } from './constants';
 import { deriveBeaconId, recoverSignerAddress } from './evm';
 import { getAll, getAllAirnodeAddresses, prune, putAll } from './in-memory-cache';
 import { ApiResponse } from './types';
-import { generateErrorResponse, getConfig, isBatchUnique } from './utils';
+import { generateErrorResponse, isBatchUnique } from './utils';
 import { batchSignedDataSchema, evmAddressSchema } from './schema';
+import { getConfig } from './config';
 
 // Accepts a batch of signed data that is first validated for consistency and data integrity errors. If there is any
 // issue during this step, the whole batch is rejected.

packages/api/src/index.ts

@@ -4,16 +4,15 @@
 // You can check how this works by following the demo from https://github.com/evanw/node-source-map-support#demos. Just
 // create a test script with/without the source map support, build the project and run the built script using node.
 import 'source-map-support/register';
-import dotenv from 'dotenv';
 import { startServer } from './server';
-import { fetchAndCacheConfig } from './utils';
 import { logger } from './logger';
-import { Config } from './schema';
+import { fetchAndCacheConfig } from './config';
 
-dotenv.config();
-
-// Fetch the config before starting the application
-fetchAndCacheConfig().then((config: Config) => {
+async function main() {
+  const config = await fetchAndCacheConfig();
   logger.info('Using configuration', config);
+
   startServer(config);
-});
+}
+
+main();

packages/api/src/logger.ts

@@ -1,15 +1,15 @@
-import { createLogger, logLevelSchema, LogConfig } from 'signed-api/common';
+import { createLogger, logConfigSchema } from 'signed-api/common';
+import { loadEnv } from './env';
 
-const logLevel = () => {
-  const res = logLevelSchema.safeParse(process.env.LOG_LEVEL || 'info');
-  return res.success ? res.data : 'info';
-};
+// We need to load the environment variables before we can use the logger. Because we want the logger to always be
+// available, we load the environment variables as a side effect during the module import.
+const env = loadEnv();
 
-const options: LogConfig = {
-  colorize: process.env.LOG_COLORIZE !== 'false',
-  enabled: process.env.LOGGER_ENABLED !== 'false',
-  minLevel: logLevel(),
-  format: process.env.LOG_FORMAT === 'json' ? 'json' : 'pretty',
-};
+const options = logConfigSchema.parse({
+  colorize: env.LOG_COLORIZE,
+  enabled: env.LOGGER_ENABLED,
+  minLevel: env.LOG_LEVEL,
+  format: env.LOG_FORMAT === 'json' ? 'json' : 'pretty',
+});
 
 export const logger = createLogger(options);

packages/api/src/schema.test.ts

@@ -1,7 +1,8 @@
 import { readFileSync } from 'fs';
 import { join } from 'path';
 import { ZodError } from 'zod';
-import { configSchema, endpointSchema, endpointsSchema } from './schema';
+import dotenv from 'dotenv';
+import { configSchema, endpointSchema, endpointsSchema, envBooleanSchema, envConfigSchema } from './schema';
 
 describe('endpointSchema', () => {
   it('validates urlPath', () => {
@@ -48,3 +49,66 @@ describe('configSchema', () => {
     expect(() => configSchema.parse(config)).not.toThrow();
   });
 });
+
+describe('env config schema', () => {
+  it('parses boolean env variable correctly', () => {
+    expect(envBooleanSchema.parse('true')).toBe(true);
+    expect(envBooleanSchema.parse('false')).toBe(false);
+
+    // Using a function to create the expected error because the error message length is too long to be inlined. The
+    // error messages is trivially stringified if propagated to the user.
+    const createExpectedError = (received: string) =>
+      new ZodError([
+        {
+          code: 'invalid_union',
+          unionErrors: [
+            new ZodError([
+              {
+                received,
+                code: 'invalid_literal',
+                expected: 'true',
+                path: [],
+                message: 'Invalid literal value, expected "true"',
+              },
+            ]),
+            new ZodError([
+              {
+                received,
+                code: 'invalid_literal',
+                expected: 'false',
+                path: [],
+                message: 'Invalid literal value, expected "false"',
+              },
+            ]),
+          ],
+          path: [],
+          message: 'Invalid input',
+        },
+      ]);
+    expect(() => envBooleanSchema.parse('')).toThrow(createExpectedError(''));
+    expect(() => envBooleanSchema.parse('off')).toThrow(createExpectedError('off'));
+  });
+
+  it('parses example env correctly', () => {
+    // Load the example configuration from the ".env.example" file
+    const env = dotenv.parse(readFileSync(join(__dirname, '../.env.example'), 'utf8'));
+
+    expect(() => envConfigSchema.parse(env)).not.toThrow();
+  });
+
+  it('AWS_REGION is set when CONFIG_SOURCE is aws-s3', () => {
+    const env = {
+      CONFIG_SOURCE: 'aws-s3',
+    };
+
+    expect(() => envConfigSchema.parse(env)).toThrow(
+      new ZodError([
+        {
+          code: 'custom',
+          message: 'The AWS_REGION must be set when CONFIG_SOURCE is "aws-s3"',
+          path: ['AWS_REGION'],
+        },
+      ])
+    );
+  });
+});

packages/api/src/schema.ts

@@ -1,4 +1,5 @@
 import { uniqBy } from 'lodash';
+import { logFormatSchema, logLevelSchema } from 'signed-api/common';
 import { z } from 'zod';
 
 export const endpointSchema = z
@@ -50,3 +51,35 @@ export type SignedData = z.infer<typeof signedDataSchema>;
 export const batchSignedDataSchema = z.array(signedDataSchema);
 
 export type BatchSignedData = z.infer<typeof batchSignedDataSchema>;
+
+export const envBooleanSchema = z.union([z.literal('true'), z.literal('false')]).transform((val) => val === 'true');
+
+// We apply default values to make it convenient to omit certain environment variables. The default values should be
+// primarily focused on users and production usage.
+export const envConfigSchema = z
+  .object({
+    LOGGER_ENABLED: envBooleanSchema.default('true'),
+    LOG_COLORIZE: envBooleanSchema.default('false'),
+    LOG_FORMAT: logFormatSchema.default('json'),
+    LOG_LEVEL: logLevelSchema.default('info'),
+
+    CONFIG_SOURCE: z.union([z.literal('local'), z.literal('aws-s3')]).default('local'),
+
+    AWS_ACCESS_KEY_ID: z.string().optional(),
+    AWS_SECRET_ACCESS_KEY: z.string().optional(),
+    AWS_REGION: z.string().optional(),
+    AWS_S3_BUCKET_NAME: z.string().optional(),
+    AWS_S3_BUCKET_PATH: z.string().optional(),
+  })
+  .strip() // We parse from ENV variables of the process which has many variables that we don't care about.
+  .superRefine((val, ctx) => {
+    if (val.CONFIG_SOURCE === 'aws-s3' && !val.AWS_REGION) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: 'The AWS_REGION must be set when CONFIG_SOURCE is "aws-s3"',
+        path: ['AWS_REGION'],
+      });
+    }
+  });
+
+export type EnvConfig = z.infer<typeof envConfigSchema>;

packages/api/src/utils.ts

@@ -1,10 +1,5 @@
-import { readFileSync } from 'fs';
-import { join } from 'path';
-import { go } from '@api3/promise-utils';
-import { S3 } from '@aws-sdk/client-s3';
 import { COMMON_HEADERS } from './constants';
-import { logger } from './logger';
-import { BatchSignedData, Config, SignedData, configSchema } from './schema';
+import { BatchSignedData, SignedData } from './schema';
 import { ApiResponse } from './types';
 
 export const isBatchUnique = (batchSignedData: BatchSignedData) => {
@@ -26,48 +21,3 @@ export const generateErrorResponse = (
 ): ApiResponse => {
   return { statusCode, headers: COMMON_HEADERS, body: JSON.stringify({ message, detail, extra }) };
 };
-
-let config: Config;
-export const getConfig = (): Config => {
-  if (!config) {
-    throw new Error(`config has not been set yet`);
-  }
-  return config;
-};
-
-export const fetchAndCacheConfig = async (): Promise<Config> => {
-  const jsonConfig = await fetchConfig();
-  config = configSchema.parse(jsonConfig);
-  return config;
-};
-
-const fetchConfig = async (): Promise<any> => {
-  const source = process.env.CONFIG_SOURCE;
-  if (!source || source === 'local') {
-    return JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8'));
-  }
-  if (source === 'aws-s3') {
-    return await fetchConfigFromS3();
-  }
-  throw new Error(`Unable to load config CONFIG_SOURCE:${source}`);
-};
-
-const fetchConfigFromS3 = async (): Promise<any> => {
-  const region = process.env.AWS_REGION!;
-  const s3 = new S3({ region });
-
-  const params = {
-    Bucket: process.env.AWS_S3_BUCKET_NAME!,
-    Key: process.env.AWS_S3_BUCKET_PATH!,
-  };
-
-  logger.info(`Fetching config from AWS S3 region:${region}...`);
-  const res = await go(() => s3.getObject(params), { retries: 1 });
-  if (!res.success) {
-    logger.error('Error fetching config from AWS S3:', res.error);
-    throw res.error;
-  }
-  logger.info('Config fetched successfully from AWS S3');
-  const stringifiedConfig = await res.data.Body!.transformToString();
-  return JSON.parse(stringifiedConfig);
-};

packages/data-pusher/README.md

@@ -16,7 +16,8 @@ To start the the pusher in dev mode run the following:
 2. `cp secrets.example.env secrets.env` - To copy the secrets.env needed for the configuration. This file is also
    ignored by git.
 3. Set the `NODARY_API_KEY` inside the secrets file.
-4. `pnpm run dev` - To run the pusher. This step assumes already running signed API as specified in the `pusher.json`
+4. `cp .env.example .env` - To copy the example environment variables. Optionally change the defaults.
+5. `pnpm run dev` - To run the pusher. This step assumes already running signed API as specified in the `pusher.json`
    configuration.
 
 ## Docker

packages/data-pusher/src/api-requests/signed-api.test.ts

@@ -1,14 +1,8 @@
 import axios from 'axios';
 import { ZodError } from 'zod';
 import { postSignedApiData, signTemplateResponses } from './signed-api';
-import {
-  config,
-  createMockedLogger,
-  signedApiResponse,
-  nodarySignedTemplateResponses,
-  nodaryTemplateResponses,
-} from '../../test/fixtures';
-import * as loggerModule from '../logger';
+import { config, signedApiResponse, nodarySignedTemplateResponses, nodaryTemplateResponses } from '../../test/fixtures';
+import { logger } from '../logger';
 import * as stateModule from '../state';
 
 describe(signTemplateResponses.name, () => {
@@ -57,8 +51,7 @@ describe(postSignedApiData.name, () => {
       })
     );
     jest.spyOn(stateModule, 'getState').mockReturnValue(state);
-    const logger = createMockedLogger();
-    jest.spyOn(loggerModule, 'logger').mockReturnValue(logger);
+    jest.spyOn(logger, 'warn');
     jest.spyOn(axios, 'post').mockResolvedValue({ youHaveNotThoughAboutThisDidYou: 'yes-I-did' });
 
     const response = await postSignedApiData(config.triggers.signedApiUpdates[0]!);
@@ -90,8 +83,7 @@ describe(postSignedApiData.name, () => {
       })
     );
     jest.spyOn(stateModule, 'getState').mockReturnValue(state);
-    const logger = createMockedLogger();
-    jest.spyOn(loggerModule, 'logger').mockReturnValue(logger);
+    jest.spyOn(logger, 'warn');
     jest.spyOn(axios, 'post').mockRejectedValue('simulated-network-error');
 
     const response = await postSignedApiData(config.triggers.signedApiUpdates[0]!);