Merge pull request #139 from narval-xyz/feature/nar-1545-app-setup-ke…

…ygen

Feature/nar 1545 app setup keygen

.github/workflows/policy-engine.yml

@@ -17,6 +17,21 @@ jobs:
     name: Build and test
 
     runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:14
+        ports:
+          - '5432:5432'
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
     steps:
       - name: Checkout
         uses: actions/checkout@master
@@ -40,6 +55,8 @@ jobs:
         shell: bash
         run: |
           make policy-engine/copy-default-env
+          make policy-engine/test/db/setup
+          make policy-engine/db/generate-types
 
       - name: Test types
         shell: bash

apps/armory/.env.test.default

@@ -1,6 +1,11 @@
-# IMPORTANT: The variables defined here will override other variables.
-# See `./apps/armory/jest.setup.ts`.
-
 NODE_ENV=test
 
+PORT=3005
+
 ARMORY_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/armory_test?schema=public"
+
+REDIS_HOST=localhost
+REDIS_PORT=6379
+
+PRICE_FEED_PRIVATE_KEY="0xc7a1b8ba040a238e36058fc5693f801d129aca9f10ed30d0133878f1b9147c01"
+HISTORICAL_TRANSFER_FEED_PRIVATE_KEY="0xf5c8f17cc09215c5038f6b8d5e557c0d98d341236307fe831efdcdd7faeef134"

apps/armory/jest.setup.ts

@@ -3,26 +3,28 @@ import fs from 'fs'
 import nock from 'nock'
 
 const testEnvFile = `${__dirname}/.env.test`
-const envFile = `${__dirname}/.env`
 
 // Ensure a test environment variable file exists because of the override config
 // loading mechanics below.
 if (!fs.existsSync(testEnvFile)) {
-  throw new Error('No .env.test file found. Please create one by running "make armory/copy-default-env".')
+  throw new Error('No .env.test file found. Please create one by running "make policy-engine/copy-default-env".')
+}
+
+// By default, dotenv always loads .env and then you can override with .env.test
+// But this is confusing, because then you have to look in multiple files to know which envs are loaded
+// So we will clear all envs and then load .env.test
+// NOTE: This will also override any CLI-declared envs (e.g. `MY_ENV=test jest`)
+for (const prop in process.env) {
+  if (Object.prototype.hasOwnProperty.call(process.env, prop)) {
+    delete process.env[prop]
+  }
 }
 
-// We don't want to have two dotenv files that are exactly the same, so we
-// override the default with .env.test.
-//
-// If a .env.test file is not found, the DATABASE_URL will fallback to the
-// default. Consequently, you'll lose your development database during the
-// integration tests teardown. Hence, the check above.
-dotenv.config({ path: envFile })
 dotenv.config({ path: testEnvFile, override: true })
 
 // Disable outgoing HTTP requests to avoid flaky tests.
 nock.disableNetConnect()
 
-// Enable local outgoing HTTP request to allow E2E tests with supertestwith
-// supertest to work.
-nock.enableNetConnect(/127.0.0.1|localhost/)
+// Enable outgoing HTTP requests to 127.0.0.1 to allow E2E tests with
+// supertestwith supertest to work.
+nock.enableNetConnect('127.0.0.1')

apps/policy-engine/.env.default

@@ -2,4 +2,12 @@ NODE_ENV=development
 
 PORT=3010
 
-ENGINE_DATABASE_URL="file:./engine-core.sqlite"
+POLICY_ENGINE_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/engine?schema=public"
+
+ENGINE_UID="local-dev-engine-instance-1"
+
+MASTER_PASSWORD="unsafe-local-dev-master-password"
+
+KEYRING_TYPE="raw"
+
+# MASTER_AWS_KMS_ARN="arn:aws:kms:us-east-2:728783560968:key/f6aa3ddb-47c3-4f31-977d-b93205bb23d1"

apps/policy-engine/.env.test.default

@@ -1,3 +1,13 @@
-# IMPORTANT: The variables defined here will override other variables.
-# See `./apps/policy-engine/jest.setup.ts`.
 NODE_ENV=test
+
+PORT=3010
+
+POLICY_ENGINE_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/engine-test?schema=public"
+
+ENGINE_UID="local-dev-engine-instance-1"
+
+MASTER_PASSWORD="unsafe-local-test-master-password"
+
+KEYRING_TYPE="raw"
+
+# MASTER_AWS_KMS_ARN="arn:aws:kms:us-east-2:728783560968:key/f6aa3ddb-47c3-4f31-977d-b93205bb23d1"

apps/policy-engine/Makefile

@@ -1,5 +1,6 @@
 POLICY_ENGINE_PROJECT_NAME := policy-engine
 POLICY_ENGINE_PROJECT_DIR := ./apps/policy-engine
+POLICY_ENGINE_DATABASE_SCHEMA := ${POLICY_ENGINE_PROJECT_DIR}/src/shared/module/persistence/schema/schema.prisma
 
 # === Start ===
 
@@ -11,11 +12,19 @@ policy-engine/start/dev:
 policy-engine/setup:
 	make policy-engine/copy-default-env
 	make policy-engine/rego/build
+	make policy-engine/db/setup
+	make policy-engine/test/db/setup
 
 policy-engine/copy-default-env:
 	cp ${POLICY_ENGINE_PROJECT_DIR}/.env.default ${POLICY_ENGINE_PROJECT_DIR}/.env
 	cp ${POLICY_ENGINE_PROJECT_DIR}/.env.test.default ${POLICY_ENGINE_PROJECT_DIR}/.env.test
 
+# === Build ===
+
+policy-engine/build/script:
+	npx tsc --project ${POLICY_ENGINE_PROJECT_DIR}/tsconfig.app.json
+	npx tsc-alias --project ${POLICY_ENGINE_PROJECT_DIR}/tsconfig.app.json
+
 # == Code format ==
 
 policy-engine/format:
@@ -30,9 +39,59 @@ policy-engine/format/check:
 policy-engine/lint/check:
 	npx nx lint ${POLICY_ENGINE_PROJECT_NAME}
 
+# === Database ===
+
+policy-engine/db/generate-types:
+	npx prisma generate \
+		--schema ${POLICY_ENGINE_DATABASE_SCHEMA}
+
+policy-engine/db/migrate:
+	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env -- \
+		prisma migrate dev \
+			--schema ${POLICY_ENGINE_DATABASE_SCHEMA}
+
+policy-engine/db/setup:
+	@echo ""
+	@echo "${TERM_GREEN}🛠️  Setting up Engine development database${TERM_NO_COLOR}"
+	@echo ""
+	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env -- \
+		prisma migrate reset \
+			--schema ${POLICY_ENGINE_DATABASE_SCHEMA} \
+			--force
+	make policy-engine/db/seed
+
+	@echo ""
+	@echo "${TERM_GREEN}🛠️  Setting up Engine test database${TERM_NO_COLOR}"
+	@echo ""
+	make policy-engine/test/db/setup
+
+policy-engine/db/create-migration:
+	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env -- \
+		prisma migrate dev \
+			--schema ${POLICY_ENGINE_DATABASE_SCHEMA} \
+			--name ${NAME}
+
+# To maintain seed data within their respective modules and then import them
+# into the main seed.ts file for execution, it's necessary to compile the
+# project and resolve its path aliases before running the vanilla JavaScript
+# seed entry point.
+policy-engine/db/seed:
+	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env -- \
+		ts-node -r tsconfig-paths/register --project ${POLICY_ENGINE_PROJECT_DIR}/tsconfig.app.json ${POLICY_ENGINE_PROJECT_DIR}/src/shared/module/persistence/seed.ts
+
+
 # === Testing ===
 
+policy-engine/test/db/setup:
+	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env.test --override -- \
+		prisma migrate reset \
+		--schema ${POLICY_ENGINE_DATABASE_SCHEMA} \
+		--skip-seed \
+		--force
+
+
 policy-engine/test/type:
+	make policy-engine/db/generate-types
 	npx tsc \
 		--project ${POLICY_ENGINE_PROJECT_DIR}/tsconfig.app.json \
 		--noEmit

apps/policy-engine/jest.setup.ts

@@ -3,21 +3,23 @@ import fs from 'fs'
 import nock from 'nock'
 
 const testEnvFile = `${__dirname}/.env.test`
-const envFile = `${__dirname}/.env`
 
 // Ensure a test environment variable file exists because of the override config
 // loading mechanics below.
 if (!fs.existsSync(testEnvFile)) {
-  throw new Error('No .env.test file found. Please create one by running "make authz/copy-default-env".')
+  throw new Error('No .env.test file found. Please create one by running "make policy-engine/copy-default-env".')
+}
+
+// By default, dotenv always loads .env and then you can override with .env.test
+// But this is confusing, because then you have to look in multiple files to know which envs are loaded
+// So we will clear all envs and then load .env.test
+// NOTE: This will also override any CLI-declared envs (e.g. `MY_ENV=test jest`)
+for (const prop in process.env) {
+  if (Object.prototype.hasOwnProperty.call(process.env, prop)) {
+    delete process.env[prop]
+  }
 }
 
-// We don't want to have two dotenv files that are exactly the same, so we
-// override the default with .env.test.
-//
-// If a .env.test file is not found, the DATABASE_URL will fallback to the
-// default. Consequently, you'll lose your development database during the
-// integration tests teardown. Hence, the check above.
-dotenv.config({ path: envFile })
 dotenv.config({ path: testEnvFile, override: true })
 
 // Disable outgoing HTTP requests to avoid flaky tests.

apps/policy-engine/src/app/__test__/e2e/admin.spec.ts

@@ -8,7 +8,10 @@ import { readFileSync, unlinkSync } from 'fs'
 import { mock } from 'jest-mock-extended'
 import request from 'supertest'
 import { AppModule } from '../../../app/app.module'
+import { EncryptionService } from '../../../encryption/core/encryption.service'
 import { load } from '../../../policy-engine.config'
+import { PersistenceModule } from '../../../shared/module/persistence/persistence.module'
+import { TestPrismaService } from '../../../shared/module/persistence/service/test-prisma.service'
 import { Organization } from '../../../shared/types/entities.types'
 import { Criterion, Then, TimeWindow } from '../../../shared/types/policy.type'
 import { EntityRepository } from '../../persistence/repository/entity.repository'
@@ -17,6 +20,7 @@ const REQUEST_HEADER_ORG_ID = 'x-org-id'
 describe('Admin Endpoints', () => {
   let app: INestApplication
   let module: TestingModule
+  let testPrismaService: TestPrismaService
 
   // TODO: Real sigs; these will NOT match the test data.
   const authentication: Signature = {
@@ -45,30 +49,40 @@ describe('Admin Endpoints', () => {
   beforeAll(async () => {
     const entityRepositoryMock = mock<EntityRepository>()
     entityRepositoryMock.fetch.mockResolvedValue(FIXTURE.ENTITIES)
+    const encryptionMock = mock<EncryptionService>()
 
     module = await Test.createTestingModule({
       imports: [
         ConfigModule.forRoot({
           load: [load],
           isGlobal: true
         }),
-        AppModule
+        AppModule,
+        PersistenceModule
       ]
     })
       .overrideProvider(EntityRepository)
       .useValue(entityRepositoryMock)
+      .overrideProvider(EncryptionService)
+      .useValue(encryptionMock)
       .compile()
 
+    testPrismaService = module.get<TestPrismaService>(TestPrismaService)
     app = module.createNestApplication()
 
     await app.init()
   })
 
   afterAll(async () => {
+    await testPrismaService.truncateAll()
     await module.close()
     await app.close()
   })
 
+  afterEach(async () => {
+    await testPrismaService.truncateAll()
+  })
+
   describe('POST /policies', () => {
     it('sets the organization policies', async () => {
       const payload = {

apps/policy-engine/src/app/app.module.ts

@@ -2,6 +2,7 @@ import { HttpModule } from '@nestjs/axios'
 import { Module, ValidationPipe } from '@nestjs/common'
 import { ConfigModule } from '@nestjs/config'
 import { APP_PIPE } from '@nestjs/core'
+import { EncryptionModule } from '../encryption/encryption.module'
 import { load } from '../policy-engine.config'
 import { AppController } from './app.controller'
 import { AppService } from './app.service'
@@ -16,7 +17,8 @@ import { EntityRepository } from './persistence/repository/entity.repository'
       load: [load],
       isGlobal: true
     }),
-    HttpModule
+    HttpModule,
+    EncryptionModule
   ],
   controllers: [AppController, AdminController],
   providers: [

apps/policy-engine/src/encryption/core/__test__/unit/encryption.service.spec.ts

@@ -0,0 +1,82 @@
+import { ConfigModule, ConfigService } from '@nestjs/config'
+import { Test } from '@nestjs/testing'
+import { mock } from 'jest-mock-extended'
+import nock from 'nock'
+import { load } from '../../../../policy-engine.config'
+import { EncryptionRepository } from '../../../persistence/repository/encryption.repository'
+import { EncryptionService } from '../../encryption.service'
+
+describe('EncryptionService', () => {
+  let service: EncryptionService
+
+  nock.enableNetConnect('kms.us-east-2.amazonaws.com:443')
+
+  beforeEach(async () => {
+    // These mocked config values matter; they're specifically tied to the mocked masterKey below
+    // If you change these, the decryption won't work & tests will fail
+    const configServiceMock = mock<ConfigService>({
+      get: jest.fn().mockImplementation((key: string) => {
+        if (key === 'keyring') {
+          return {
+            type: 'raw',
+            masterPassword: 'unsafe-local-dev-master-password'
+          }
+        }
+        if (key === 'engine.id') {
+          return 'local-dev-engine-instance-1'
+        }
+      })
+    })
+
+    const encryptionRepositoryMock = {
+      getEngine: jest.fn().mockImplementation(() =>
+        Promise.resolve({
+          // unencryptedMasterKey: dfd9cc70f1ad02d19e0efa020d82f557022f59ca6bedbec1df38e8fd37ae3bb9
+          masterKey:
+            '0205785d67737fa3bae8eb249cf8d3baed5942f1677d8c98b4cdeef55560a3bcf510bd008d00030003617070000d61726d6f72792d656e67696e6500156177732d63727970746f2d7075626c69632d6b657900444177336764324b6e58646f512f2b76745347367031444442384d65766d61434b324c7861426e65476a315531537777526b376b4d366868752f707a446f48724c77773d3d0007707572706f7365000f646174612d656e6372797074696f6e000100146e617276616c2e61726d6f72792e656e67696e65002561726d6f72792e656e67696e652e6b656b000000800000000c8a92a7c9deb43316f6c29e8d0030132d63c7337c9888a06b638966e83056a0575958b42588b7aed999b9659e6d4bc5bed4664d91fae0b14d48917e00cdbb02000010000749ed0ed3616b7990f9e73f5a42eb46dc182002612e33dcb8e3c7d4759184c46ce3f0893a87ac15257d53097ac5d74affffffff00000001000000000000000000000001000000205d7209b51db8cf8264b9065add71a8514dc26baa6987d8a0a3acb1c4a2503b0f3b7c974a35ed234c1b94668736cd8bfa00673065023100a5d8d192e9802649dab86af6e00ab6d7472533e85dfe1006cb8bd9ef2472d15096fa42e742d18cb92530c762c3bd44d40230350299b42feaa1149c6ad1b25add24c30b3bf1c08263b96df0d43e2ad3e19802872e792040f1faf3d0a73bca6fb067ca',
+          id: 'test-engine-id'
+        })
+      )
+    }
+    const moduleRef = await Test.createTestingModule({
+      imports: [
+        ConfigModule.forRoot({
+          load: [load],
+          isGlobal: true
+        })
+      ],
+      providers: [
+        EncryptionService,
+        {
+          provide: EncryptionRepository,
+          useValue: encryptionRepositoryMock
+        },
+        {
+          provide: ConfigService,
+          useValue: configServiceMock // use the mock ConfigService
+        }
+      ]
+    }).compile()
+
+    service = moduleRef.get<EncryptionService>(EncryptionService)
+    if (service.onApplicationBootstrap) {
+      await service.onApplicationBootstrap()
+    }
+  })
+
+  it('should encrypt then decrypt successfully, with a string', async () => {
+    const data = 'Hello World'
+    const encrypted = await service.encrypt(data)
+    const decrypted = await service.decrypt(encrypted)
+
+    expect(decrypted.toString('utf-8')).toBe(data)
+  })
+
+  it('should encrypt then decrypt successfully, with a buffer from a hexstring', async () => {
+    const data = 'dfd9cc70f1ad02d19e0efa020d82f557022f59ca6bedbec1df38e8fd37ae3bb9'
+    const encrypted = await service.encrypt(Buffer.from(data, 'hex'))
+    const decrypted = await service.decrypt(encrypted)
+
+    expect(decrypted.toString('hex')).toBe(data)
+  })
+})