create internal endpoint for ingest source file get

core/internal/ingest/get.js

@@ -0,0 +1,90 @@
+const router = require('express').Router()
+const { httpErrorHandler } = require('../../../common/error-handling/http')
+const streamSourceFileDao = require('../../stream-source-files/dao')
+const streamSegmentDao = require('../../stream-segments/dao')
+const Converter = require('../../../common/converter')
+const rolesService = require('../../roles/dao')
+const { ForbiddenError, EmptyResultError } = require('../../../common/error-handling/errors')
+
+/**
+ * @swagger
+ *
+ * /streams/{id}/stream-source-file:
+ *   get:
+ *     summary: Get a stream source file belonging to a stream with segments
+ *     tags:
+ *       - internal
+ *     parameters:
+ *       - name: filename
+ *         description: Filename
+ *         in: query
+ *         type: string
+ *       - name: sha1_checksum
+ *         description: List of sha1 checksums
+ *         in: query
+ *         type: string
+ *       - name: sha1_checksum
+ *         description: List of sha1 checksums
+ *         in: query
+ *         type: string
+ *       - name: start
+ *         description: File timestamp
+ *         in: query
+ *         type: string
+ *       - name: fields
+ *         description: Customize included fields and relations
+ *         in: query
+ *         type: array
+ *     responses:
+ *       200:
+ *         description: Stream source file (lite) object
+ *         content:
+ *           application/json:
+ *             schema:
+*                $ref: '#/components/schemas/StreamSourceFileLiteWithAvailability'
+ *       400:
+ *         description: Invalid query parameters
+ *       404:
+ *         description: Stream not found
+ */
+router.get('/:id/stream-source-file', function (req, res) {
+  const user = req.rfcx.auth_token_info
+  const streamId = req.params.id
+  const converter = new Converter(req.query, {}, true)
+  converter.convert('sha1_checksum').toString()
+  converter.convert('start').toMomentUtc()
+  converter.convert('filename').optional().toString()
+  converter.convert('fields').optional().toArray()
+
+  return converter.validate()
+    .then(async (params) => {
+      if (!user.has_system_role && !await rolesService.hasPermission(rolesService.READ, user, streamId, rolesService.STREAM)) {
+        throw new ForbiddenError('You do not have permission to access this stream.')
+      }
+      const filters = {
+        streamIds: [req.params.id],
+        sha1Checksums: [params.sha1Checksum]
+      }
+      if (params.filename) { filters.filename = [params.filename] }
+      const options = {
+        fields: params.fields
+      }
+      const data = await streamSourceFileDao.query(filters, options)
+      if (!data.results.length) {
+        throw new EmptyResultError('Stream source file not found')
+      }
+      const streamSourceFile = data.results[0]
+      const segmentsData = await streamSegmentDao.query({
+        streamId: req.params.id,
+        start: params.start.clone().subtract('1', 'minute'),
+        end: params.start.clone().add('1', 'day'),
+        streamSourceFileId: streamSourceFile.id
+      }, { fields: ['start', 'availability'] })
+      streamSourceFile.availability = streamSourceFileDao.calcAvailability(segmentsData.results)
+      streamSourceFile.segments = segmentsData.results
+      return res.json(streamSourceFile)
+    })
+    .catch(httpErrorHandler(req, res, 'Failed getting stream source file'))
+})
+
+module.exports = router

core/internal/ingest/stream.int.test.js → core/internal/ingest/post.int.test.js

@@ -1,6 +1,6 @@
 const request = require('supertest')
 const moment = require('moment')
-const routes = require('./stream')
+const routes = require('./post')
 const models = require('../../_models')
 const { truncateNonBase, expressApp, seedValues, muteConsole } = require('../../../common/testing/sequelize')
 
@@ -40,7 +40,7 @@ function resetTestData () {
       audio_codec: 'flac',
       sha1_checksum: 'e427f7bf6c589b4856d5f51691d159366d74266',
       meta: {
-        Artist: 'Topher White',
+        Artist: 'John Doe',
         Album: 'Rainforest Connection'
       }
     },

core/internal/routes.js

@@ -23,7 +23,8 @@ module.exports = {
     require('./explorer/stream-classifications')
   ],
   ingest: [
-    require('./ingest/stream')
+    require('./ingest/get'),
+    require('./ingest/post')
   ],
   prediction: [
     require('./prediction/classifier-deployments'),

core/stream-source-files/stream.int.test.js

@@ -66,7 +66,6 @@ describe('GET stream-source-file/:id', () => {
     expect(response.body.filename).toBe(sourceFile.filename)
     expect(response.body.duration).toBe(sourceFile.duration)
     expect(response.body.sample_rate).toBe(sourceFile.sample_rate)
-    expect(response.body.availability).toBe(segment1.availability)
   })
   test('receives stream source with 1 unavailable stream segment assigned', async () => {
     const { audioFileFormat, audioCodec, fileExtension, stream } = await commonSetup()
@@ -81,7 +80,6 @@ describe('GET stream-source-file/:id', () => {
     expect(response.body.filename).toBe(sourceFile.filename)
     expect(response.body.duration).toBe(sourceFile.duration)
     expect(response.body.sample_rate).toBe(sourceFile.sample_rate)
-    expect(response.body.availability).toBe(segment1.availability)
   })
   test('receives stream source with 1 unavailable and 1 available stream segments assigned', async () => {
     const { audioFileFormat, audioCodec, fileExtension, stream } = await commonSetup()
@@ -97,7 +95,6 @@ describe('GET stream-source-file/:id', () => {
     expect(response.body.filename).toBe(sourceFile.filename)
     expect(response.body.duration).toBe(sourceFile.duration)
     expect(response.body.sample_rate).toBe(sourceFile.sample_rate)
-    expect(response.body.availability).toBe(segment1.availability)
   })
   test('receives stream source with 1 unavailable and 2 available stream segments assigned', async () => {
     const { audioFileFormat, audioCodec, fileExtension, stream } = await commonSetup()
@@ -114,7 +111,6 @@ describe('GET stream-source-file/:id', () => {
     expect(response.body.filename).toBe(sourceFile.filename)
     expect(response.body.duration).toBe(sourceFile.duration)
     expect(response.body.sample_rate).toBe(sourceFile.sample_rate)
-    expect(response.body.availability).toBe(segment1.availability)
   })
   test('receives stream source with 1 unavailable and 1 available and 1 cold storage stream segments assigned', async () => {
     const { audioFileFormat, audioCodec, fileExtension, stream } = await commonSetup()
@@ -131,7 +127,6 @@ describe('GET stream-source-file/:id', () => {
     expect(response.body.filename).toBe(sourceFile.filename)
     expect(response.body.duration).toBe(sourceFile.duration)
     expect(response.body.sample_rate).toBe(sourceFile.sample_rate)
-    expect(response.body.availability).toBe(segment1.availability)
   })
   test('receives stream source with 1 unavailable and 1 cold storage stream segments assigned', async () => {
     const { audioFileFormat, audioCodec, fileExtension, stream } = await commonSetup()
@@ -147,13 +142,12 @@ describe('GET stream-source-file/:id', () => {
     expect(response.body.filename).toBe(sourceFile.filename)
     expect(response.body.duration).toBe(sourceFile.duration)
     expect(response.body.sample_rate).toBe(sourceFile.sample_rate)
-    expect(response.body.availability).toBe(segment1.availability)
   })
   test('receives stream source with 1 available and 1 cold storage stream segments assigned', async () => {
     const { audioFileFormat, audioCodec, fileExtension, stream } = await commonSetup()
     const sourceFile = await models.StreamSourceFile.create({ stream_id: stream.id, filename: '20210726_101010.wav', duration: 60, sample_count: 720000, sample_rate: 12000, channels_count: 1, bit_rate: 1, audio_codec_id: audioCodec.id, audio_file_format_id: audioFileFormat.id, sha1_checksum: 'b37530881c7ffd9edfd8f7feb131ae4563e3759f' })
     const segment1 = await models.StreamSegment.create({ stream_id: stream.id, start: '2021-07-26T10:11:10Z', end: '2021-07-26T10:12:10Z', stream_source_file_id: sourceFile.id, sample_count: 720000, file_extension_id: fileExtension.id, availability: 1 })
-    const segment2 = await models.StreamSegment.create({ stream_id: stream.id, start: '2021-07-26T10:12:10Z', end: '2021-07-26T10:13:10Z', stream_source_file_id: sourceFile.id, sample_count: 720000, file_extension_id: fileExtension.id, availability: 2 })
+    await models.StreamSegment.create({ stream_id: stream.id, start: '2021-07-26T10:12:10Z', end: '2021-07-26T10:13:10Z', stream_source_file_id: sourceFile.id, sample_count: 720000, file_extension_id: fileExtension.id, availability: 2 })
     await models.StreamSegment.create({ stream_id: stream.id, start: '2021-07-26T10:12:10Z', end: '2021-07-26T10:13:10Z', stream_source_file_id: sourceFile.id, sample_count: 720000, file_extension_id: fileExtension.id, availability: 2 })
 
     const response = await request(app).get(`/${stream.id}/stream-source-file`).query({ sha1_checksum: sourceFile.sha1_checksum, start: segment1.start })
@@ -163,7 +157,6 @@ describe('GET stream-source-file/:id', () => {
     expect(response.body.filename).toBe(sourceFile.filename)
     expect(response.body.duration).toBe(sourceFile.duration)
     expect(response.body.sample_rate).toBe(sourceFile.sample_rate)
-    expect(response.body.availability).toBe(segment2.availability)
   })
   test('receives stream source with 2 available stream segments assigned', async () => {
     const { audioFileFormat, audioCodec, fileExtension, stream } = await commonSetup()
@@ -179,6 +172,5 @@ describe('GET stream-source-file/:id', () => {
     expect(response.body.filename).toBe(sourceFile.filename)
     expect(response.body.duration).toBe(sourceFile.duration)
     expect(response.body.sample_rate).toBe(sourceFile.sample_rate)
-    expect(response.body.availability).toBe(segment1.availability)
   })
 })

core/stream-source-files/stream.js

@@ -4,7 +4,6 @@ const dao = require('./dao')
 const Converter = require('../../common/converter')
 const { ForbiddenError, EmptyResultError } = require('../../common/error-handling/errors')
 const rolesService = require('../roles/dao')
-const streamSegmentDao = require('../stream-segments/dao/index')
 
 /**
  * @swagger
@@ -41,7 +40,7 @@ const streamSegmentDao = require('../stream-segments/dao/index')
  *         content:
  *           application/json:
  *             schema:
-*                $ref: '#/components/schemas/StreamSourceFileLiteWithAvailability'
+*                $ref: '#/components/schemas/StreamSourceFileLite'
  *       400:
  *         description: Invalid query parameters
  *       404:
@@ -74,13 +73,6 @@ router.get('/:id/stream-source-file', function (req, res) {
         throw new EmptyResultError('Stream source file not found')
       }
       const streamSourceFile = data.results[0]
-      const segmentsData = await streamSegmentDao.query({
-        streamId: req.params.id,
-        start: params.start.clone().subtract('1', 'minute'),
-        end: params.start.clone().add('1', 'day'),
-        streamSourceFileId: streamSourceFile.id
-      }, { fields: ['availability'] })
-      streamSourceFile.availability = dao.calcAvailability(segmentsData.results)
       return res.json(streamSourceFile)
     })
     .catch(httpErrorHandler(req, res, 'Failed getting stream source file'))