From ce26d4826c0a85a52cd4721c8037e05ec6338728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 1 Oct 2023 11:10:59 +0200 Subject: [PATCH] Improve validation for ENVs --- packages/api/.env.example | 3 +- packages/api/README.md | 3 +- packages/api/src/config.ts | 54 +++++++++++++++ packages/api/src/env.ts | 19 ++++++ packages/api/src/handlers.test.ts | 4 +- packages/api/src/handlers.ts | 3 +- packages/api/src/index.ts | 15 ++--- packages/api/src/logger.ts | 22 +++---- packages/api/src/schema.test.ts | 66 ++++++++++++++++++- packages/api/src/schema.ts | 33 ++++++++++ packages/api/src/utils.ts | 52 +-------------- packages/data-pusher/README.md | 3 +- .../src/api-requests/signed-api.test.ts | 16 ++--- packages/data-pusher/src/index.ts | 5 +- packages/data-pusher/src/logger.ts | 23 ++++--- packages/data-pusher/src/validation/env.ts | 19 ++++++ packages/data-pusher/src/validation/schema.ts | 16 +++++ packages/data-pusher/test/fixtures.ts | 11 ---- 18 files changed, 250 insertions(+), 117 deletions(-) create mode 100644 packages/api/src/config.ts create mode 100644 packages/api/src/env.ts create mode 100644 packages/data-pusher/src/validation/env.ts diff --git a/packages/api/.env.example b/packages/api/.env.example index 7f892e22..f68cf4b0 100644 --- a/packages/api/.env.example +++ b/packages/api/.env.example @@ -13,5 +13,4 @@ AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_REGION= AWS_S3_BUCKET_NAME= -AWS_S3_BUCKET_PATH="path/to/config/inside/bucket/signed-api.json" - +AWS_S3_BUCKET_PATH= diff --git a/packages/api/README.md b/packages/api/README.md index 33a5dc55..6acecc43 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -6,7 +6,8 @@ A service for storing and accessing signed data. It provides endpoints to handle 1. `cp config/signed-api.example.json config/signed-api.json` - To create a config file from the example one. Optionally change the defaults. -2. `pnpm run dev` - To start the API server. The port number can be configured in the configuration file. +2. `cp .env.example .env` - To copy the example environment variables. Optionally change the defaults. +3. `pnpm run dev` - To start the API server. The port number can be configured in the configuration file. ## Deployment diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts new file mode 100644 index 00000000..e38ec71b --- /dev/null +++ b/packages/api/src/config.ts @@ -0,0 +1,54 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { go } from '@api3/promise-utils'; +import { S3 } from '@aws-sdk/client-s3'; +import { logger } from './logger'; +import { Config, configSchema } from './schema'; +import { loadEnv } from './env'; + +let config: Config | undefined; + +export const getConfig = (): Config => { + if (!config) throw new Error(`Config has not been set yet`); + + return config; +}; + +export const fetchAndCacheConfig = async (): Promise => { + const jsonConfig = await fetchConfig(); + config = configSchema.parse(jsonConfig); + return config; +}; + +const fetchConfig = async (): Promise => { + const env = loadEnv(); + const source = env.CONFIG_SOURCE; + if (!source || source === 'local') { + return JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8')); + } + if (source === 'aws-s3') { + return await fetchConfigFromS3(); + } + throw new Error(`Unable to load config CONFIG_SOURCE:${source}`); +}; + +const fetchConfigFromS3 = async (): Promise => { + const env = loadEnv(); + const region = env.AWS_REGION!; // Validated by environment variables schema. + const s3 = new S3({ region }); + + const params = { + Bucket: env.AWS_S3_BUCKET_NAME, + Key: env.AWS_S3_BUCKET_PATH, + }; + + logger.info(`Fetching config from AWS S3 region:${region}...`); + const res = await go(() => s3.getObject(params), { retries: 1 }); + if (!res.success) { + logger.error('Error fetching config from AWS S3:', res.error); + throw res.error; + } + logger.info('Config fetched successfully from AWS S3'); + const stringifiedConfig = await res.data.Body!.transformToString(); + return JSON.parse(stringifiedConfig); +}; diff --git a/packages/api/src/env.ts b/packages/api/src/env.ts new file mode 100644 index 00000000..1144c0f4 --- /dev/null +++ b/packages/api/src/env.ts @@ -0,0 +1,19 @@ +import { join } from 'path'; +import dotenv from 'dotenv'; +import { EnvConfig, envConfigSchema } from './schema'; + +let env: EnvConfig | undefined; + +export const loadEnv = () => { + if (env) return env; + + dotenv.config({ path: join(__dirname, '../.env') }); + + const parseResult = envConfigSchema.safeParse(process.env); + if (!parseResult.success) { + throw new Error(`Invalid environment variables:\n, ${JSON.stringify(parseResult.error.format())}`); + } + + env = parseResult.data; + return env; +}; diff --git a/packages/api/src/handlers.test.ts b/packages/api/src/handlers.test.ts index 50eb3c9d..ad6ba954 100644 --- a/packages/api/src/handlers.test.ts +++ b/packages/api/src/handlers.test.ts @@ -2,7 +2,7 @@ import { readFileSync } from 'fs'; import { join } from 'path'; import { omit } from 'lodash'; import * as cacheModule from './cache'; -import * as utilsModule from './utils'; +import * as configModule from './config'; import { batchInsertData, getData, listAirnodeAddresses } from './handlers'; import { createSignedData, generateRandomWallet } from '../test/utils'; @@ -12,7 +12,7 @@ afterEach(() => { beforeEach(() => { jest - .spyOn(utilsModule, 'getConfig') + .spyOn(configModule, 'getConfig') .mockImplementation(() => JSON.parse(readFileSync(join(__dirname, '../config/signed-api.example.json'), 'utf8'))); }); diff --git a/packages/api/src/handlers.ts b/packages/api/src/handlers.ts index 9746dec1..7a18307f 100644 --- a/packages/api/src/handlers.ts +++ b/packages/api/src/handlers.ts @@ -4,8 +4,9 @@ import { CACHE_HEADERS, COMMON_HEADERS } from './constants'; import { deriveBeaconId, recoverSignerAddress } from './evm'; import { getAll, getAllAirnodeAddresses, prune, putAll } from './in-memory-cache'; import { ApiResponse } from './types'; -import { generateErrorResponse, getConfig, isBatchUnique } from './utils'; +import { generateErrorResponse, isBatchUnique } from './utils'; import { batchSignedDataSchema, evmAddressSchema } from './schema'; +import { getConfig } from './config'; // Accepts a batch of signed data that is first validated for consistency and data integrity errors. If there is any // issue during this step, the whole batch is rejected. diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 84cd9f29..ea256869 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -4,16 +4,15 @@ // You can check how this works by following the demo from https://github.com/evanw/node-source-map-support#demos. Just // create a test script with/without the source map support, build the project and run the built script using node. import 'source-map-support/register'; -import dotenv from 'dotenv'; import { startServer } from './server'; -import { fetchAndCacheConfig } from './utils'; import { logger } from './logger'; -import { Config } from './schema'; +import { fetchAndCacheConfig } from './config'; -dotenv.config(); - -// Fetch the config before starting the application -fetchAndCacheConfig().then((config: Config) => { +async function main() { + const config = await fetchAndCacheConfig(); logger.info('Using configuration', config); + startServer(config); -}); +} + +main(); diff --git a/packages/api/src/logger.ts b/packages/api/src/logger.ts index 196f9eef..c4ad4e46 100644 --- a/packages/api/src/logger.ts +++ b/packages/api/src/logger.ts @@ -1,15 +1,15 @@ -import { createLogger, logLevelSchema, LogConfig } from 'signed-api/common'; +import { createLogger, logConfigSchema } from 'signed-api/common'; +import { loadEnv } from './env'; -const logLevel = () => { - const res = logLevelSchema.safeParse(process.env.LOG_LEVEL || 'info'); - return res.success ? res.data : 'info'; -}; +// We need to load the environment variables before we can use the logger. Because we want the logger to always be +// available, we load the environment variables as a side effect during the module import. +const env = loadEnv(); -const options: LogConfig = { - colorize: process.env.LOG_COLORIZE !== 'false', - enabled: process.env.LOGGER_ENABLED !== 'false', - minLevel: logLevel(), - format: process.env.LOG_FORMAT === 'json' ? 'json' : 'pretty', -}; +const options = logConfigSchema.parse({ + colorize: env.LOG_COLORIZE, + enabled: env.LOGGER_ENABLED, + minLevel: env.LOG_LEVEL, + format: env.LOG_FORMAT === 'json' ? 'json' : 'pretty', +}); export const logger = createLogger(options); diff --git a/packages/api/src/schema.test.ts b/packages/api/src/schema.test.ts index d073f206..93df6898 100644 --- a/packages/api/src/schema.test.ts +++ b/packages/api/src/schema.test.ts @@ -1,7 +1,8 @@ import { readFileSync } from 'fs'; import { join } from 'path'; import { ZodError } from 'zod'; -import { configSchema, endpointSchema, endpointsSchema } from './schema'; +import dotenv from 'dotenv'; +import { configSchema, endpointSchema, endpointsSchema, envBooleanSchema, envConfigSchema } from './schema'; describe('endpointSchema', () => { it('validates urlPath', () => { @@ -48,3 +49,66 @@ describe('configSchema', () => { expect(() => configSchema.parse(config)).not.toThrow(); }); }); + +describe('env config schema', () => { + it('parses boolean env variable correctly', () => { + expect(envBooleanSchema.parse('true')).toBe(true); + expect(envBooleanSchema.parse('false')).toBe(false); + + // Using a function to create the expected error because the error message length is too long to be inlined. The + // error messages is trivially stringified if propagated to the user. + const createExpectedError = (received: string) => + new ZodError([ + { + code: 'invalid_union', + unionErrors: [ + new ZodError([ + { + received, + code: 'invalid_literal', + expected: 'true', + path: [], + message: 'Invalid literal value, expected "true"', + }, + ]), + new ZodError([ + { + received, + code: 'invalid_literal', + expected: 'false', + path: [], + message: 'Invalid literal value, expected "false"', + }, + ]), + ], + path: [], + message: 'Invalid input', + }, + ]); + expect(() => envBooleanSchema.parse('')).toThrow(createExpectedError('')); + expect(() => envBooleanSchema.parse('off')).toThrow(createExpectedError('off')); + }); + + it('parses example env correctly', () => { + // Load the example configuration from the ".env.example" file + const env = dotenv.parse(readFileSync(join(__dirname, '../.env.example'), 'utf8')); + + expect(() => envConfigSchema.parse(env)).not.toThrow(); + }); + + it('AWS_REGION is set when CONFIG_SOURCE is aws-s3', () => { + const env = { + CONFIG_SOURCE: 'aws-s3', + }; + + expect(() => envConfigSchema.parse(env)).toThrow( + new ZodError([ + { + code: 'custom', + message: 'The AWS_REGION must be set when CONFIG_SOURCE is "aws-s3"', + path: ['AWS_REGION'], + }, + ]) + ); + }); +}); diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 6a52b2f1..66f64538 100644 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1,4 +1,5 @@ import { uniqBy } from 'lodash'; +import { logFormatSchema, logLevelSchema } from 'signed-api/common'; import { z } from 'zod'; export const endpointSchema = z @@ -50,3 +51,35 @@ export type SignedData = z.infer; export const batchSignedDataSchema = z.array(signedDataSchema); export type BatchSignedData = z.infer; + +export const envBooleanSchema = z.union([z.literal('true'), z.literal('false')]).transform((val) => val === 'true'); + +// We apply default values to make it convenient to omit certain environment variables. The default values should be +// primarily focused on users and production usage. +export const envConfigSchema = z + .object({ + LOGGER_ENABLED: envBooleanSchema.default('true'), + LOG_COLORIZE: envBooleanSchema.default('false'), + LOG_FORMAT: logFormatSchema.default('json'), + LOG_LEVEL: logLevelSchema.default('info'), + + CONFIG_SOURCE: z.union([z.literal('local'), z.literal('aws-s3')]).default('local'), + + AWS_ACCESS_KEY_ID: z.string().optional(), + AWS_SECRET_ACCESS_KEY: z.string().optional(), + AWS_REGION: z.string().optional(), + AWS_S3_BUCKET_NAME: z.string().optional(), + AWS_S3_BUCKET_PATH: z.string().optional(), + }) + .strip() // We parse from ENV variables of the process which has many variables that we don't care about. + .superRefine((val, ctx) => { + if (val.CONFIG_SOURCE === 'aws-s3' && !val.AWS_REGION) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'The AWS_REGION must be set when CONFIG_SOURCE is "aws-s3"', + path: ['AWS_REGION'], + }); + } + }); + +export type EnvConfig = z.infer; diff --git a/packages/api/src/utils.ts b/packages/api/src/utils.ts index 35348f2c..37a51de1 100644 --- a/packages/api/src/utils.ts +++ b/packages/api/src/utils.ts @@ -1,10 +1,5 @@ -import { readFileSync } from 'fs'; -import { join } from 'path'; -import { go } from '@api3/promise-utils'; -import { S3 } from '@aws-sdk/client-s3'; import { COMMON_HEADERS } from './constants'; -import { logger } from './logger'; -import { BatchSignedData, Config, SignedData, configSchema } from './schema'; +import { BatchSignedData, SignedData } from './schema'; import { ApiResponse } from './types'; export const isBatchUnique = (batchSignedData: BatchSignedData) => { @@ -26,48 +21,3 @@ export const generateErrorResponse = ( ): ApiResponse => { return { statusCode, headers: COMMON_HEADERS, body: JSON.stringify({ message, detail, extra }) }; }; - -let config: Config; -export const getConfig = (): Config => { - if (!config) { - throw new Error(`config has not been set yet`); - } - return config; -}; - -export const fetchAndCacheConfig = async (): Promise => { - const jsonConfig = await fetchConfig(); - config = configSchema.parse(jsonConfig); - return config; -}; - -const fetchConfig = async (): Promise => { - const source = process.env.CONFIG_SOURCE; - if (!source || source === 'local') { - return JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8')); - } - if (source === 'aws-s3') { - return await fetchConfigFromS3(); - } - throw new Error(`Unable to load config CONFIG_SOURCE:${source}`); -}; - -const fetchConfigFromS3 = async (): Promise => { - const region = process.env.AWS_REGION!; - const s3 = new S3({ region }); - - const params = { - Bucket: process.env.AWS_S3_BUCKET_NAME!, - Key: process.env.AWS_S3_BUCKET_PATH!, - }; - - logger.info(`Fetching config from AWS S3 region:${region}...`); - const res = await go(() => s3.getObject(params), { retries: 1 }); - if (!res.success) { - logger.error('Error fetching config from AWS S3:', res.error); - throw res.error; - } - logger.info('Config fetched successfully from AWS S3'); - const stringifiedConfig = await res.data.Body!.transformToString(); - return JSON.parse(stringifiedConfig); -}; diff --git a/packages/data-pusher/README.md b/packages/data-pusher/README.md index b520a025..98815732 100644 --- a/packages/data-pusher/README.md +++ b/packages/data-pusher/README.md @@ -16,7 +16,8 @@ To start the the pusher in dev mode run the following: 2. `cp secrets.example.env secrets.env` - To copy the secrets.env needed for the configuration. This file is also ignored by git. 3. Set the `NODARY_API_KEY` inside the secrets file. -4. `pnpm run dev` - To run the pusher. This step assumes already running signed API as specified in the `pusher.json` +4. `cp .env.example .env` - To copy the example environment variables. Optionally change the defaults. +5. `pnpm run dev` - To run the pusher. This step assumes already running signed API as specified in the `pusher.json` configuration. ## Docker diff --git a/packages/data-pusher/src/api-requests/signed-api.test.ts b/packages/data-pusher/src/api-requests/signed-api.test.ts index c8c4b286..2c30ce0a 100644 --- a/packages/data-pusher/src/api-requests/signed-api.test.ts +++ b/packages/data-pusher/src/api-requests/signed-api.test.ts @@ -1,14 +1,8 @@ import axios from 'axios'; import { ZodError } from 'zod'; import { postSignedApiData, signTemplateResponses } from './signed-api'; -import { - config, - createMockedLogger, - signedApiResponse, - nodarySignedTemplateResponses, - nodaryTemplateResponses, -} from '../../test/fixtures'; -import * as loggerModule from '../logger'; +import { config, signedApiResponse, nodarySignedTemplateResponses, nodaryTemplateResponses } from '../../test/fixtures'; +import { logger } from '../logger'; import * as stateModule from '../state'; describe(signTemplateResponses.name, () => { @@ -57,8 +51,7 @@ describe(postSignedApiData.name, () => { }) ); jest.spyOn(stateModule, 'getState').mockReturnValue(state); - const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); + jest.spyOn(logger, 'warn'); jest.spyOn(axios, 'post').mockResolvedValue({ youHaveNotThoughAboutThisDidYou: 'yes-I-did' }); const response = await postSignedApiData(config.triggers.signedApiUpdates[0]!); @@ -90,8 +83,7 @@ describe(postSignedApiData.name, () => { }) ); jest.spyOn(stateModule, 'getState').mockReturnValue(state); - const logger = createMockedLogger(); - jest.spyOn(loggerModule, 'logger').mockReturnValue(logger); + jest.spyOn(logger, 'warn'); jest.spyOn(axios, 'post').mockRejectedValue('simulated-network-error'); const response = await postSignedApiData(config.triggers.signedApiUpdates[0]!); diff --git a/packages/data-pusher/src/index.ts b/packages/data-pusher/src/index.ts index e5e3454c..ebce301c 100644 --- a/packages/data-pusher/src/index.ts +++ b/packages/data-pusher/src/index.ts @@ -4,15 +4,12 @@ // You can check how this works by following the demo from https://github.com/evanw/node-source-map-support#demos. Just // create a test script with/without the source map support, build the project and run the built script using node. import 'source-map-support/register'; -import dotenv from 'dotenv'; import { loadConfig } from './validation/config'; import { initiateFetchingBeaconData } from './fetch-beacon-data'; import { initiateUpdatingSignedApi } from './update-signed-api'; import { initializeState } from './state'; -dotenv.config(); - -export async function main() { +async function main() { const config = await loadConfig(); initializeState(config); diff --git a/packages/data-pusher/src/logger.ts b/packages/data-pusher/src/logger.ts index 489b421e..86d77fc6 100644 --- a/packages/data-pusher/src/logger.ts +++ b/packages/data-pusher/src/logger.ts @@ -1,16 +1,15 @@ -import { createLogger, logLevelSchema, LogConfig } from 'signed-api/common'; +import { createLogger, logConfigSchema } from 'signed-api/common'; +import { loadEnv } from './validation/env'; -const logLevel = () => { - const res = logLevelSchema.safeParse(process.env.LOG_LEVEL || 'info'); - return res.success ? res.data : 'info'; -}; +// We need to load the environment variables before we can use the logger. Because we want the logger to always be +// available, we load the environment variables as a side effect during the module import. +const env = loadEnv(); -const options: LogConfig = { - colorize: process.env.LOG_COLORIZE !== 'false', - enabled: process.env.LOGGER_ENABLED !== 'false', - minLevel: logLevel(), - format: process.env.LOG_FORMAT === 'json' ? 'json' : 'pretty', -}; +const options = logConfigSchema.parse({ + colorize: env.LOG_COLORIZE, + enabled: env.LOGGER_ENABLED, + minLevel: env.LOG_LEVEL, + format: env.LOG_FORMAT === 'json' ? 'json' : 'pretty', +}); export const logger = createLogger(options); - diff --git a/packages/data-pusher/src/validation/env.ts b/packages/data-pusher/src/validation/env.ts new file mode 100644 index 00000000..fbe5a1a8 --- /dev/null +++ b/packages/data-pusher/src/validation/env.ts @@ -0,0 +1,19 @@ +import { join } from 'path'; +import dotenv from 'dotenv'; +import { EnvConfig, envConfigSchema } from './schema'; + +let env: EnvConfig | undefined; + +export const loadEnv = () => { + if (env) return env; + + dotenv.config({ path: join(__dirname, '../../.env') }); + + const parseResult = envConfigSchema.safeParse(process.env); + if (!parseResult.success) { + throw new Error(`Invalid environment variables:\n, ${JSON.stringify(parseResult.error.format())}`); + } + + env = parseResult.data; + return env; +}; diff --git a/packages/data-pusher/src/validation/schema.ts b/packages/data-pusher/src/validation/schema.ts index 81faba3c..db0a0a73 100644 --- a/packages/data-pusher/src/validation/schema.ts +++ b/packages/data-pusher/src/validation/schema.ts @@ -5,6 +5,7 @@ import { oisSchema, OIS, Endpoint as oisEndpoint } from '@api3/ois'; import { config } from '@api3/airnode-validator'; import * as abi from '@api3/airnode-abi'; import * as node from '@api3/airnode-node'; +import { logFormatSchema, logLevelSchema } from 'signed-api/common'; import { preProcessApiSpecifications } from '../unexported-airnode-features/api-specification-processing'; export const limiterConfig = z.object({ minTime: z.number(), maxConcurrent: z.number() }); @@ -366,3 +367,18 @@ export const signedApiResponseSchema = z .strict(); export type SignedApiResponse = z.infer; + +export const envBooleanSchema = z.union([z.literal('true'), z.literal('false')]).transform((val) => val === 'true'); + +// We apply default values to make it convenient to omit certain environment variables. The default values should be +// primarily focused on users and production usage. +export const envConfigSchema = z + .object({ + LOGGER_ENABLED: envBooleanSchema.default('true'), + LOG_COLORIZE: envBooleanSchema.default('false'), + LOG_FORMAT: logFormatSchema.default('json'), + LOG_LEVEL: logLevelSchema.default('info'), + }) + .strip(); // We parse from ENV variables of the process which has many variables that we don't care about. + +export type EnvConfig = z.infer; diff --git a/packages/data-pusher/test/fixtures.ts b/packages/data-pusher/test/fixtures.ts index f250c713..fa800739 100644 --- a/packages/data-pusher/test/fixtures.ts +++ b/packages/data-pusher/test/fixtures.ts @@ -1,21 +1,10 @@ import { PerformApiCallSuccess } from '@api3/airnode-node/dist/src/api'; import { ApiCallErrorResponse } from '@api3/airnode-node'; -import { Logger } from 'signed-api/common'; import { AxiosResponse } from 'axios'; import { Config } from '../src/validation/schema'; import { SignedResponse } from '../src/api-requests/signed-api'; import { TemplateResponse } from '../src/api-requests/data-provider'; -export const createMockedLogger = (): Logger => { - return { - debug: jest.fn(), - warn: jest.fn(), - error: jest.fn(), - info: jest.fn(), - child: jest.fn(), - }; -}; - export const config: Config = { walletMnemonic: 'diamond result history offer forest diagram crop armed stumble orchard stage glance', rateLimiting: { maxDirectGatewayConcurrency: 25, minDirectGatewayTime: 10 },