Merge remote-tracking branch 'origin/main' into policy-devtool

.github/workflows/policy-engine.yml

@@ -17,6 +17,21 @@ jobs:
     name: Build and test
 
     runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:14
+        ports:
+          - '5432:5432'
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
     steps:
       - name: Checkout
         uses: actions/checkout@master
@@ -40,6 +55,8 @@ jobs:
         shell: bash
         run: |
           make policy-engine/copy-default-env
+          make policy-engine/test/db/setup
+          make policy-engine/db/generate-types
 
       - name: Test types
         shell: bash

apps/armory/.env.test.default

@@ -1,6 +1,11 @@
-# IMPORTANT: The variables defined here will override other variables.
-# See `./apps/armory/jest.setup.ts`.
-
 NODE_ENV=test
 
+PORT=3005
+
 ARMORY_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/armory_test?schema=public"
+
+REDIS_HOST=localhost
+REDIS_PORT=6379
+
+PRICE_FEED_PRIVATE_KEY="0xc7a1b8ba040a238e36058fc5693f801d129aca9f10ed30d0133878f1b9147c01"
+HISTORICAL_TRANSFER_FEED_PRIVATE_KEY="0xf5c8f17cc09215c5038f6b8d5e557c0d98d341236307fe831efdcdd7faeef134"

apps/armory/jest.setup.ts

@@ -3,26 +3,28 @@ import fs from 'fs'
 import nock from 'nock'
 
 const testEnvFile = `${__dirname}/.env.test`
-const envFile = `${__dirname}/.env`
 
 // Ensure a test environment variable file exists because of the override config
 // loading mechanics below.
 if (!fs.existsSync(testEnvFile)) {
-  throw new Error('No .env.test file found. Please create one by running "make armory/copy-default-env".')
+  throw new Error('No .env.test file found. Please create one by running "make policy-engine/copy-default-env".')
+}
+
+// By default, dotenv always loads .env and then you can override with .env.test
+// But this is confusing, because then you have to look in multiple files to know which envs are loaded
+// So we will clear all envs and then load .env.test
+// NOTE: This will also override any CLI-declared envs (e.g. `MY_ENV=test jest`)
+for (const prop in process.env) {
+  if (Object.prototype.hasOwnProperty.call(process.env, prop)) {
+    delete process.env[prop]
+  }
 }
 
-// We don't want to have two dotenv files that are exactly the same, so we
-// override the default with .env.test.
-//
-// If a .env.test file is not found, the DATABASE_URL will fallback to the
-// default. Consequently, you'll lose your development database during the
-// integration tests teardown. Hence, the check above.
-dotenv.config({ path: envFile })
 dotenv.config({ path: testEnvFile, override: true })
 
 // Disable outgoing HTTP requests to avoid flaky tests.
 nock.disableNetConnect()
 
-// Enable local outgoing HTTP request to allow E2E tests with supertestwith
-// supertest to work.
-nock.enableNetConnect(/127.0.0.1|localhost/)
+// Enable outgoing HTTP requests to 127.0.0.1 to allow E2E tests with
+// supertestwith supertest to work.
+nock.enableNetConnect('127.0.0.1')

apps/policy-engine/.env.default

@@ -2,4 +2,12 @@ NODE_ENV=development
 
 PORT=3010
 
-ENGINE_DATABASE_URL="file:./engine-core.sqlite"
+POLICY_ENGINE_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/engine?schema=public"
+
+ENGINE_UID="local-dev-engine-instance-1"
+
+MASTER_PASSWORD="unsafe-local-dev-master-password"
+
+KEYRING_TYPE="raw"
+
+# MASTER_AWS_KMS_ARN="arn:aws:kms:us-east-2:728783560968:key/f6aa3ddb-47c3-4f31-977d-b93205bb23d1"

apps/policy-engine/.env.test.default

@@ -1,3 +1,13 @@
-# IMPORTANT: The variables defined here will override other variables.
-# See `./apps/policy-engine/jest.setup.ts`.
 NODE_ENV=test
+
+PORT=3010
+
+POLICY_ENGINE_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/engine-test?schema=public"
+
+ENGINE_UID="local-dev-engine-instance-1"
+
+MASTER_PASSWORD="unsafe-local-test-master-password"
+
+KEYRING_TYPE="raw"
+
+# MASTER_AWS_KMS_ARN="arn:aws:kms:us-east-2:728783560968:key/f6aa3ddb-47c3-4f31-977d-b93205bb23d1"

apps/policy-engine/Makefile

@@ -1,5 +1,6 @@
 POLICY_ENGINE_PROJECT_NAME := policy-engine
 POLICY_ENGINE_PROJECT_DIR := ./apps/policy-engine
+POLICY_ENGINE_DATABASE_SCHEMA := ${POLICY_ENGINE_PROJECT_DIR}/src/shared/module/persistence/schema/schema.prisma
 
 # === Start ===
 
@@ -11,11 +12,19 @@ policy-engine/start/dev:
 policy-engine/setup:
 	make policy-engine/copy-default-env
 	make policy-engine/rego/build
+	make policy-engine/db/setup
+	make policy-engine/test/db/setup
 
 policy-engine/copy-default-env:
 	cp ${POLICY_ENGINE_PROJECT_DIR}/.env.default ${POLICY_ENGINE_PROJECT_DIR}/.env
 	cp ${POLICY_ENGINE_PROJECT_DIR}/.env.test.default ${POLICY_ENGINE_PROJECT_DIR}/.env.test
 
+# === Build ===
+
+policy-engine/build/script:
+	npx tsc --project ${POLICY_ENGINE_PROJECT_DIR}/tsconfig.app.json
+	npx tsc-alias --project ${POLICY_ENGINE_PROJECT_DIR}/tsconfig.app.json
+
 # == Code format ==
 
 policy-engine/format:
@@ -30,9 +39,59 @@ policy-engine/format/check:
 policy-engine/lint/check:
 	npx nx lint ${POLICY_ENGINE_PROJECT_NAME}
 
+# === Database ===
+
+policy-engine/db/generate-types:
+	npx prisma generate \
+		--schema ${POLICY_ENGINE_DATABASE_SCHEMA}
+
+policy-engine/db/migrate:
+	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env -- \
+		prisma migrate dev \
+			--schema ${POLICY_ENGINE_DATABASE_SCHEMA}
+
+policy-engine/db/setup:
+	@echo ""
+	@echo "${TERM_GREEN}🛠️  Setting up Engine development database${TERM_NO_COLOR}"
+	@echo ""
+	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env -- \
+		prisma migrate reset \
+			--schema ${POLICY_ENGINE_DATABASE_SCHEMA} \
+			--force
+	make policy-engine/db/seed
+
+	@echo ""
+	@echo "${TERM_GREEN}🛠️  Setting up Engine test database${TERM_NO_COLOR}"
+	@echo ""
+	make policy-engine/test/db/setup
+
+policy-engine/db/create-migration:
+	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env -- \
+		prisma migrate dev \
+			--schema ${POLICY_ENGINE_DATABASE_SCHEMA} \
+			--name ${NAME}
+
+# To maintain seed data within their respective modules and then import them
+# into the main seed.ts file for execution, it's necessary to compile the
+# project and resolve its path aliases before running the vanilla JavaScript
+# seed entry point.
+policy-engine/db/seed:
+	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env -- \
+		ts-node -r tsconfig-paths/register --project ${POLICY_ENGINE_PROJECT_DIR}/tsconfig.app.json ${POLICY_ENGINE_PROJECT_DIR}/src/shared/module/persistence/seed.ts
+
+
 # === Testing ===
 
+policy-engine/test/db/setup:
+	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env.test --override -- \
+		prisma migrate reset \
+		--schema ${POLICY_ENGINE_DATABASE_SCHEMA} \
+		--skip-seed \
+		--force
+
+
 policy-engine/test/type:
+	make policy-engine/db/generate-types
 	npx tsc \
 		--project ${POLICY_ENGINE_PROJECT_DIR}/tsconfig.app.json \
 		--noEmit

apps/policy-engine/jest.setup.ts

@@ -3,21 +3,23 @@ import fs from 'fs'
 import nock from 'nock'
 
 const testEnvFile = `${__dirname}/.env.test`
-const envFile = `${__dirname}/.env`
 
 // Ensure a test environment variable file exists because of the override config
 // loading mechanics below.
 if (!fs.existsSync(testEnvFile)) {
-  throw new Error('No .env.test file found. Please create one by running "make authz/copy-default-env".')
+  throw new Error('No .env.test file found. Please create one by running "make policy-engine/copy-default-env".')
+}
+
+// By default, dotenv always loads .env and then you can override with .env.test
+// But this is confusing, because then you have to look in multiple files to know which envs are loaded
+// So we will clear all envs and then load .env.test
+// NOTE: This will also override any CLI-declared envs (e.g. `MY_ENV=test jest`)
+for (const prop in process.env) {
+  if (Object.prototype.hasOwnProperty.call(process.env, prop)) {
+    delete process.env[prop]
+  }
 }
 
-// We don't want to have two dotenv files that are exactly the same, so we
-// override the default with .env.test.
-//
-// If a .env.test file is not found, the DATABASE_URL will fallback to the
-// default. Consequently, you'll lose your development database during the
-// integration tests teardown. Hence, the check above.
-dotenv.config({ path: envFile })
 dotenv.config({ path: testEnvFile, override: true })
 
 // Disable outgoing HTTP requests to avoid flaky tests.

apps/policy-engine/src/app/__test__/e2e/admin.spec.ts

@@ -8,15 +8,19 @@ import { readFileSync, unlinkSync } from 'fs'
 import { mock } from 'jest-mock-extended'
 import request from 'supertest'
 import { AppModule } from '../../../app/app.module'
+import { EncryptionService } from '../../../encryption/core/encryption.service'
+import { load } from '../../../policy-engine.config'
+import { PersistenceModule } from '../../../shared/module/persistence/persistence.module'
+import { TestPrismaService } from '../../../shared/module/persistence/service/test-prisma.service'
 import { Organization } from '../../../shared/types/entities.types'
 import { Criterion, Then, TimeWindow } from '../../../shared/types/policy.type'
-import { load } from '../../app.config'
 import { EntityRepository } from '../../persistence/repository/entity.repository'
 
 const REQUEST_HEADER_ORG_ID = 'x-org-id'
 describe('Admin Endpoints', () => {
   let app: INestApplication
   let module: TestingModule
+  let testPrismaService: TestPrismaService
 
   // TODO: Real sigs; these will NOT match the test data.
   const authentication: Signature = {
@@ -45,30 +49,40 @@ describe('Admin Endpoints', () => {
   beforeAll(async () => {
     const entityRepositoryMock = mock<EntityRepository>()
     entityRepositoryMock.fetch.mockResolvedValue(FIXTURE.ENTITIES)
+    const encryptionMock = mock<EncryptionService>()
 
     module = await Test.createTestingModule({
       imports: [
         ConfigModule.forRoot({
           load: [load],
           isGlobal: true
         }),
-        AppModule
+        AppModule,
+        PersistenceModule
       ]
     })
       .overrideProvider(EntityRepository)
       .useValue(entityRepositoryMock)
+      .overrideProvider(EncryptionService)
+      .useValue(encryptionMock)
       .compile()
 
+    testPrismaService = module.get<TestPrismaService>(TestPrismaService)
     app = module.createNestApplication()
 
     await app.init()
   })
 
   afterAll(async () => {
+    await testPrismaService.truncateAll()
     await module.close()
     await app.close()
   })
 
+  afterEach(async () => {
+    await testPrismaService.truncateAll()
+  })
+
   describe('POST /policies', () => {
     it('sets the organization policies', async () => {
       const payload = {

apps/policy-engine/src/app/app.module.ts

@@ -2,7 +2,8 @@ import { HttpModule } from '@nestjs/axios'
 import { Module, ValidationPipe } from '@nestjs/common'
 import { ConfigModule } from '@nestjs/config'
 import { APP_PIPE } from '@nestjs/core'
-import { load } from './app.config'
+import { EncryptionModule } from '../encryption/encryption.module'
+import { load } from '../policy-engine.config'
 import { AppController } from './app.controller'
 import { AppService } from './app.service'
 import { AdminService } from './core/admin.service'
@@ -16,7 +17,8 @@ import { EntityRepository } from './persistence/repository/entity.repository'
       load: [load],
       isGlobal: true
     }),
-    HttpModule
+    HttpModule,
+    EncryptionModule
   ],
   controllers: [AppController, AdminController],
   providers: [

apps/policy-engine/src/app/persistence/repository/__test__/unit/tenant.repository.spec.ts

@@ -0,0 +1,60 @@
+import { DataStoreConfiguration } from '@narval/policy-engine-shared'
+import { Test } from '@nestjs/testing'
+import { KeyValueRepository } from '../../../../../shared/module/key-value/core/repository/key-value.repository'
+import { KeyValueService } from '../../../../../shared/module/key-value/core/service/key-value.service'
+import { InMemoryKeyValueRepository } from '../../../../../shared/module/key-value/persistence/repository/in-memory-key-value.repository'
+import { Tenant } from '../../../../../shared/types/domain.type'
+import { TenantRepository } from '../../../repository/tenant.repository'
+
+describe(TenantRepository.name, () => {
+  let repository: TenantRepository
+  let inMemoryKeyValueRepository: InMemoryKeyValueRepository
+
+  beforeEach(async () => {
+    inMemoryKeyValueRepository = new InMemoryKeyValueRepository()
+
+    const module = await Test.createTestingModule({
+      providers: [
+        KeyValueService,
+        TenantRepository,
+        {
+          provide: KeyValueRepository,
+          useValue: inMemoryKeyValueRepository
+        }
+      ]
+    }).compile()
+
+    repository = module.get<TenantRepository>(TenantRepository)
+  })
+
+  describe('create', () => {
+    const now = new Date()
+
+    const dataStoreConfiguration: DataStoreConfiguration = {
+      dataUrl: 'a-url-that-doesnt-need-to-exist-for-the-purpose-of-this-test',
+      signatureUrl: 'a-url-that-doesnt-need-to-exist-for-the-purpose-of-this-test',
+      keys: []
+    }
+
+    const tenant: Tenant = {
+      clientId: 'test-client-id',
+      clientSecret: 'test-client-secret',
+      dataStore: {
+        entity: dataStoreConfiguration,
+        policy: dataStoreConfiguration
+      },
+      createdAt: now,
+      updatedAt: now
+    }
+
+    it('creates a new tenant', async () => {
+      await repository.create(tenant)
+
+      const value = await inMemoryKeyValueRepository.get(repository.getKey(tenant.clientId))
+      const actualTenant = await repository.findByClientId(tenant.clientId)
+
+      expect(value).not.toEqual(null)
+      expect(tenant).toEqual(actualTenant)
+    })
+  })
+})