Implement data store service for the file system and HTTP (#156)

README.md

@@ -64,11 +64,11 @@ below to generate a project of your choice.
 
 ```bash
 # Generate an standard JavaScript library.
- npx nx g @nrwl/workspace:lib
- # Generate an NestJS library.
- npx nx g @nx/nest:library
- # Generate an NestJS application.
- npx nx g @nx/nest:application
+npx nx g @nrwl/workspace:lib
+# Generate an NestJS library.
+npx nx g @nx/nest:library
+# Generate an NestJS application.
+npx nx g @nx/nest:application
 ```
 
 For more information about code generation, please refer to the [NX

apps/policy-engine/src/app/core/exception/data-store.exception.ts

@@ -0,0 +1,3 @@
+import { ApplicationException } from '../../../shared/exception/application.exception'
+
+export class DataStoreException extends ApplicationException {}

apps/policy-engine/src/app/core/factory/data-store-repository.factory.ts

@@ -0,0 +1,33 @@
+import { HttpStatus, Injectable } from '@nestjs/common'
+import { FileSystemDataStoreRepository } from '../../persistence/repository/file-system-data-store.repository'
+import { HttpDataStoreRepository } from '../../persistence/repository/http-data-store.repository'
+import { DataStoreException } from '../exception/data-store.exception'
+import { DataStoreRepository } from '../repository/data-store.repository'
+
+@Injectable()
+export class DataStoreRepositoryFactory {
+  constructor(
+    private fileSystemRepository: FileSystemDataStoreRepository,
+    private httpRepository: HttpDataStoreRepository
+  ) {}
+
+  getRepository(url: string): DataStoreRepository {
+    switch (this.getProtocol(url)) {
+      case 'file':
+        return this.fileSystemRepository
+      case 'http':
+      case 'https':
+        return this.httpRepository
+      default:
+        throw new DataStoreException({
+          message: 'Data store URL protocol not supported',
+          suggestedHttpStatusCode: HttpStatus.UNPROCESSABLE_ENTITY,
+          context: { url }
+        })
+    }
+  }
+
+  private getProtocol(url: string): string {
+    return url.split(':')[0]
+  }
+}

apps/policy-engine/src/app/core/repository/data-store.repository.ts

@@ -0,0 +1,3 @@
+export interface DataStoreRepository {
+  fetch<Data>(url: string): Promise<Data>
+}

apps/policy-engine/src/app/core/service/__test__/integration/data-store.service.spec.ts

@@ -0,0 +1,57 @@
+import { DataStoreConfiguration, EntityData, EntitySignature, FIXTURE } from '@narval/policy-engine-shared'
+import { HttpModule } from '@nestjs/axios'
+import { HttpStatus } from '@nestjs/common'
+import { Test } from '@nestjs/testing'
+import nock from 'nock'
+import { FileSystemDataStoreRepository } from '../../../../../app/persistence/repository/file-system-data-store.repository'
+import { HttpDataStoreRepository } from '../../../../../app/persistence/repository/http-data-store.repository'
+import { withTempJsonFile } from '../../../../../shared/testing/with-temp-json-file.testing'
+import { DataStoreRepositoryFactory } from '../../../factory/data-store-repository.factory'
+import { DataStoreService } from '../../data-store.service'
+
+describe(DataStoreService.name, () => {
+  let service: DataStoreService
+
+  const remoteDataStoreUrl = 'http://9.9.9.9:9000'
+
+  const entityDataStore: EntityData = {
+    entity: {
+      data: FIXTURE.ENTITIES
+    }
+  }
+
+  const entitySignatureStore: EntitySignature = {
+    entity: {
+      signature: 'test-signature'
+    }
+  }
+
+  beforeEach(async () => {
+    const module = await Test.createTestingModule({
+      imports: [HttpModule],
+      providers: [DataStoreService, DataStoreRepositoryFactory, HttpDataStoreRepository, FileSystemDataStoreRepository]
+    }).compile()
+
+    service = module.get<DataStoreService>(DataStoreService)
+  })
+
+  describe('fetch', () => {
+    it('fetches data and signature from distinct stores', async () => {
+      nock(remoteDataStoreUrl).get('/').reply(HttpStatus.OK, entityDataStore)
+
+      await withTempJsonFile(JSON.stringify(entitySignatureStore), async (path) => {
+        const url = `file://${path}`
+        const config: DataStoreConfiguration = {
+          dataUrl: remoteDataStoreUrl,
+          signatureUrl: url,
+          keys: []
+        }
+
+        const { entity } = await service.fetch(config)
+
+        expect(entity.data).toEqual(entityDataStore.entity.data)
+        expect(entity.signature).toEqual(entitySignatureStore.entity.signature)
+      })
+    })
+  })
+})

apps/policy-engine/src/app/core/service/data-store.service.ts

@@ -0,0 +1,51 @@
+import { DataStoreConfiguration, entityDataSchema, entitySignatureSchema } from '@narval/policy-engine-shared'
+import { HttpStatus, Injectable } from '@nestjs/common'
+import { ZodObject, z } from 'zod'
+import { DataStoreException } from '../exception/data-store.exception'
+import { DataStoreRepositoryFactory } from '../factory/data-store-repository.factory'
+
+@Injectable()
+export class DataStoreService {
+  constructor(private dataStoreRepositoryFactory: DataStoreRepositoryFactory) {}
+
+  async fetch(config: DataStoreConfiguration) {
+    const [entityData, entitySignature] = await Promise.all([
+      this.fetchByUrl(config.dataUrl, entityDataSchema),
+      this.fetchByUrl(config.signatureUrl, entitySignatureSchema)
+    ])
+
+    return {
+      entity: {
+        data: entityData.entity.data,
+        signature: entitySignature.entity.signature
+      }
+    }
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private async fetchByUrl<DataSchema extends ZodObject<any>>(
+    url: string,
+    schema: DataSchema
+  ): Promise<z.infer<typeof schema>> {
+    const data = await this.dataStoreRepositoryFactory.getRepository(url).fetch(url)
+    const result = schema.safeParse(data)
+
+    if (result.success) {
+      return result.data
+    }
+
+    throw new DataStoreException({
+      message: 'Invalid store schema',
+      suggestedHttpStatusCode: HttpStatus.UNPROCESSABLE_ENTITY,
+      context: {
+        ...(schema.description ? { schema: schema.description } : {}),
+        url,
+        errors: result.error.errors.map(({ path, message, code }) => ({
+          path,
+          code,
+          message
+        }))
+      }
+    })
+  }
+}

apps/policy-engine/src/app/persistence/repository/__test__/integration/file-system-data-store.repository.spec.ts

@@ -0,0 +1,45 @@
+import { EntityData, FIXTURE } from '@narval/policy-engine-shared'
+import { Test } from '@nestjs/testing'
+import { withTempJsonFile } from '../../../../../shared/testing/with-temp-json-file.testing'
+import { DataStoreException } from '../../../../core/exception/data-store.exception'
+import { FileSystemDataStoreRepository } from '../../file-system-data-store.repository'
+
+describe(FileSystemDataStoreRepository.name, () => {
+  let repository: FileSystemDataStoreRepository
+
+  const entityData: EntityData = {
+    entity: {
+      data: FIXTURE.ENTITIES
+    }
+  }
+
+  beforeEach(async () => {
+    const module = await Test.createTestingModule({
+      providers: [FileSystemDataStoreRepository]
+    }).compile()
+
+    repository = module.get<FileSystemDataStoreRepository>(FileSystemDataStoreRepository)
+  })
+
+  describe('fetch', () => {
+    it('fetches data from a data source in the local file system', async () => {
+      await withTempJsonFile(JSON.stringify(entityData), async (path) => {
+        const data = await repository.fetch(`file://${path}`)
+
+        expect(data).toEqual(entityData)
+      })
+    })
+
+    it('throws a DataStoreException when file does not exist', async () => {
+      const notFoundDataStoreUrl = 'file://./this-file-does-not-exist-in-the-file-system.json'
+
+      await expect(() => repository.fetch(notFoundDataStoreUrl)).rejects.toThrow(DataStoreException)
+    })
+
+    it('throws a DataStoreException when the json is invalid', async () => {
+      await withTempJsonFile('[ invalid }', async (path: string) => {
+        await expect(() => repository.fetch(`file://${path}`)).rejects.toThrow(DataStoreException)
+      })
+    })
+  })
+})

apps/policy-engine/src/app/persistence/repository/__test__/integration/http-data-store.repository.spec.ts

@@ -0,0 +1,46 @@
+import { EntityData, FIXTURE } from '@narval/policy-engine-shared'
+import { HttpModule } from '@nestjs/axios'
+import { HttpStatus } from '@nestjs/common'
+import { Test } from '@nestjs/testing'
+import nock from 'nock'
+import { DataStoreException } from '../../../../../app/core/exception/data-store.exception'
+import { HttpDataStoreRepository } from '../../http-data-store.repository'
+
+describe(HttpDataStoreRepository.name, () => {
+  let repository: HttpDataStoreRepository
+
+  const dataStoreHost = 'http://some.host:3010'
+  const dataStoreEndpoint = '/data-store/entities'
+  const dataStoreUrl = dataStoreHost + dataStoreEndpoint
+
+  const entityData: EntityData = {
+    entity: {
+      data: FIXTURE.ENTITIES
+    }
+  }
+
+  beforeEach(async () => {
+    const module = await Test.createTestingModule({
+      imports: [HttpModule],
+      providers: [HttpDataStoreRepository]
+    }).compile()
+
+    repository = module.get<HttpDataStoreRepository>(HttpDataStoreRepository)
+  })
+
+  describe('fetch', () => {
+    it('fetches data from a remote data source via http protocol', async () => {
+      nock(dataStoreHost).get(dataStoreEndpoint).reply(HttpStatus.OK, entityData)
+
+      const data = await repository.fetch(dataStoreUrl)
+
+      expect(data).toEqual(entityData)
+    })
+
+    it('throws a DataStoreException when it fails to fetch', async () => {
+      nock(dataStoreHost).get(dataStoreEndpoint).reply(HttpStatus.INTERNAL_SERVER_ERROR, {})
+
+      await expect(() => repository.fetch(dataStoreUrl)).rejects.toThrow(DataStoreException)
+    })
+  })
+})

apps/policy-engine/src/app/persistence/repository/__test__/unit/tenant.repository.spec.ts

@@ -79,5 +79,11 @@ describe(TenantRepository.name, () => {
       expect(value).not.toEqual(null)
       expect(tenant).toEqual(actualTenant)
     })
+
+    it('indexes the new tenant', async () => {
+      await repository.create(tenant)
+
+      expect(await repository.getTenantIndex()).toEqual([tenant.clientId])
+    })
   })
 })

apps/policy-engine/src/app/persistence/repository/file-system-data-store.repository.ts

@@ -0,0 +1,49 @@
+import { HttpStatus, Injectable } from '@nestjs/common'
+import fs from 'fs/promises'
+import { DataStoreException } from '../../core/exception/data-store.exception'
+import { DataStoreRepository } from '../../core/repository/data-store.repository'
+
+@Injectable()
+export class FileSystemDataStoreRepository implements DataStoreRepository {
+  async fetch<Data>(url: string): Promise<Data> {
+    const path = this.getPath(url)
+
+    if (await this.exists(path)) {
+      return this.read(path) as Data
+    }
+
+    throw new DataStoreException({
+      message: 'Data store file does not exist in the instance host',
+      suggestedHttpStatusCode: HttpStatus.NOT_FOUND,
+      context: { url }
+    })
+  }
+
+  private async read(path: string) {
+    try {
+      const content = await fs.readFile(path, 'utf-8')
+
+      return JSON.parse(content)
+    } catch (error) {
+      throw new DataStoreException({
+        message: 'Unable to parse data store file into JSON',
+        suggestedHttpStatusCode: HttpStatus.UNPROCESSABLE_ENTITY,
+        origin: error
+      })
+    }
+  }
+
+  private async exists(path: string): Promise<boolean> {
+    try {
+      await fs.stat(path)
+
+      return true
+    } catch (error) {
+      return false
+    }
+  }
+
+  private getPath(url: string): string {
+    return url.replace('file://', '')
+  }
+}

apps/policy-engine/src/app/persistence/repository/http-data-store.repository.ts

@@ -0,0 +1,26 @@
+import { HttpService } from '@nestjs/axios'
+import { HttpStatus, Injectable } from '@nestjs/common'
+import { catchError, lastValueFrom, map } from 'rxjs'
+import { DataStoreException } from '../../core/exception/data-store.exception'
+import { DataStoreRepository } from '../../core/repository/data-store.repository'
+
+@Injectable()
+export class HttpDataStoreRepository implements DataStoreRepository {
+  constructor(private httpService: HttpService) {}
+
+  fetch<Data>(url: string): Promise<Data> {
+    return lastValueFrom(
+      this.httpService.get<Data>(url).pipe(
+        map((response) => response.data),
+        catchError((error) => {
+          throw new DataStoreException({
+            message: 'Unable to fetch remote data source via HTTP',
+            suggestedHttpStatusCode: HttpStatus.UNPROCESSABLE_ENTITY,
+            context: { url },
+            origin: error
+          })
+        })
+      )
+    )
+  }
+}