diff --git a/noncore/_models/db.js b/noncore/_models/db.js index cc6cfdfc5..e279d1bc5 100644 --- a/noncore/_models/db.js +++ b/noncore/_models/db.js @@ -1,3 +1,4 @@ const { sequelize, Sequelize, options } = require('../../common/db')('noncore') +const coreDb = require('../../common/db')('core') -module.exports = { sequelize, Sequelize, options } +module.exports = { sequelize, Sequelize, options, coreSequelize: coreDb.sequelize, coreOptions: coreDb.options } diff --git a/noncore/_models/index.js b/noncore/_models/index.js index 0908b8e56..e22e5a857 100644 --- a/noncore/_models/index.js +++ b/noncore/_models/index.js @@ -1,4 +1,4 @@ -const { sequelize, Sequelize, options } = require('./db') +const { sequelize, Sequelize, options, coreSequelize, coreOptions } = require('./db') const models = { GuardianSite: require('./guardian-sites/guardian-site')(sequelize, Sequelize), @@ -55,6 +55,10 @@ const models = { ShortLink: require('./misc/shortlink')(sequelize, Sequelize) } +const coreModels = { + StreamSegment: require('../../core/_models/streams/stream-segment')(coreSequelize, Sequelize) +} + // Create associations Object.keys(models).forEach(function (modelName) { if ('associate' in models[modelName]) { @@ -62,4 +66,4 @@ Object.keys(models).forEach(function (modelName) { } }) -module.exports = { ...models, sequelize, Sequelize, options } +module.exports = { ...models, sequelize, Sequelize, options, ...coreModels, coreOptions } diff --git a/noncore/_utils/rfcx-mqtt/mqtt-checkin-router.js b/noncore/_utils/rfcx-mqtt/mqtt-checkin-router.js index 60e8fddce..cd260bd44 100644 --- a/noncore/_utils/rfcx-mqtt/mqtt-checkin-router.js +++ b/noncore/_utils/rfcx-mqtt/mqtt-checkin-router.js @@ -44,6 +44,9 @@ function onMessageCheckin (data, messageId) { .then((checkInObj) => { return checkInDatabase.syncGuardianPrefs(checkInObj) }) + .then((checkInObj) => { + return checkInDatabase.createDbAudio(checkInObj) + }) .then((checkInObj) => { return checkInDatabase.createDbScreenShot(checkInObj) }) @@ -62,10 +65,6 @@ function onMessageCheckin (data, messageId) { } return Promise.resolve(checkInObj) }) - // Create audio db after getting path from ingest - .then((checkInObj) => { - return checkInDatabase.createDbAudio(checkInObj) - }) .then((checkInObj) => { return checkInDatabase.finalizeCheckIn(checkInObj) }) diff --git a/noncore/_utils/rfcx-mqtt/mqtt-database.js b/noncore/_utils/rfcx-mqtt/mqtt-database.js index 7a01795cf..f4238de23 100644 --- a/noncore/_utils/rfcx-mqtt/mqtt-database.js +++ b/noncore/_utils/rfcx-mqtt/mqtt-database.js @@ -354,7 +354,7 @@ exports.checkInDatabase = { site_id: checkInObj.db.dbGuardian.site_id, check_in_id: checkInObj.db.dbCheckIn.id, sha1_checksum: checkInObj.audio.meta.sha1CheckSum, - url: `s3://${checkInObj.audio.meta.ingestBucket}/${checkInObj.audio.meta.ingestPath}`, + url: null, encode_duration: checkInObj.audio.meta.encodeDuration, measured_at: checkInObj.audio.meta.measuredAt, measured_at_local: moment.tz(checkInObj.audio.meta.measuredAt, (checkInObj.db.dbSite.timezone || 'UTC')).format('YYYY-MM-DDTHH:mm:ss.SSS'), diff --git a/noncore/_utils/rfcx-mqtt/mqtt-streams.js b/noncore/_utils/rfcx-mqtt/mqtt-streams.js index 8d1bdcc66..6d5d6e20b 100644 --- a/noncore/_utils/rfcx-mqtt/mqtt-streams.js +++ b/noncore/_utils/rfcx-mqtt/mqtt-streams.js @@ -19,8 +19,7 @@ async function ingestGuardianAudio (checkInObj) { targetBitrate: checkInObj.audio.meta.bitRate, sampleRate: checkInObj.audio.meta.sampleRate }) - checkInObj.audio.meta.ingestPath = uploadData.path - checkInObj.audio.meta.ingestBucket = uploadData.bucket + await S3Service.putObject(checkInObj.audio.filePath, uploadData.path, uploadData.bucket) assetUtils.deleteLocalFileFromFileSystem(checkInObj.audio.filePath) return checkInObj diff --git a/noncore/views/v1/models/guardian-audio.js b/noncore/views/v1/models/guardian-audio.js index 365d5ae8f..3114d4a28 100644 --- a/noncore/views/v1/models/guardian-audio.js +++ b/noncore/views/v1/models/guardian-audio.js @@ -4,9 +4,8 @@ const moment = require('moment-timezone') const fs = require('fs') const random = require('../../../../common/crypto/random') const audioUtils = require('../../../_utils/rfcx-audio').audioUtils -const assetUtils = require('../../../_utils/internal-rfcx/asset-utils').assetUtils const validation = require('../../../_utils/misc/validation') -const { GuardianSite, Guardian, GuardianAudioFormat } = require('../../../_models') +const { GuardianSite, Guardian, GuardianAudioFormat, StreamSegment } = require('../../../_models') exports.models = { @@ -19,7 +18,7 @@ exports.models = { { model: Guardian, as: 'Guardian', - attributes: ['guid', 'shortname'] + attributes: ['guid', 'shortname', 'stream_id'] }, { model: GuardianAudioFormat, @@ -28,7 +27,7 @@ exports.models = { } ], - guardianAudioFile: function (req, res, dbRow) { + guardianAudioFile: async function (req, res, dbRow) { const outputFileExtension = req.rfcx.content_type const outputFileName = dbRow.guid + '.' + outputFileExtension const isOutputEnhanced = (outputFileExtension === 'mp3') @@ -37,10 +36,13 @@ exports.models = { const queryParams = parsePermittedQueryParams(req.query, clipDurationFull) // auto-generate the asset filepath if it's not stored in the url column + const segment = await StreamSegment.findOne({ where: { stream_id: dbRow.stream_id, start: dbRow.measured_at } }) + const dateTimeString = dbRow.measured_at.toISOString().substr(0, 19).replace(/:/g, '-') + const audioPath = `/${dateTimeString.substr(0, 4)}/${dateTimeString.substr(5, 2)}/${dateTimeString.substr(8, 2)}/${dbRow.stream_id}/${segment.id}.${dbRow.Format.file_extension}` const audioStorageUrl = (dbRow.url == null) - ? 's3://' + process.env.ASSET_BUCKET_AUDIO + assetUtils.getGuardianAssetStoragePath('audio', dbRow.measured_at, dbRow.Guardian.guid, dbRow.Format.file_extension) + ? 's3://' + process.env.INGEST_BUCKET + audioPath : dbRow.url - + console.info(audioStorageUrl) audioUtils.cacheSourceAudio(audioStorageUrl) .then(function ({ sourceFilePath }) { if ((dbRow.Format.file_extension === outputFileExtension) && @@ -80,14 +82,17 @@ exports.models = { }) }, - guardianAudioSpectrogram: function (req, res, dbRow) { + guardianAudioSpectrogram: async function (req, res, dbRow) { const tmpFilePath = process.env.CACHE_DIRECTORY + 'ffmpeg/' + random.randomString(32) const queryParams = parsePermittedQueryParams(req.query, (dbRow.capture_sample_count / dbRow.Format.sample_rate)) // auto-generate the asset filepath if it's not stored in the url column + const segment = await StreamSegment.findOne({ where: { stream_id: dbRow.stream_id, start: dbRow.measured_at } }) + const dateTimeString = dbRow.measured_at.toISOString().substr(0, 19).replace(/:/g, '-') + const audioPath = `/${dateTimeString.substr(0, 4)}/${dateTimeString.substr(5, 2)}/${dateTimeString.substr(8, 2)}/${dbRow.stream_id}/${segment.id}.${dbRow.Format.file_extension}` const audioStorageUrl = (dbRow.url == null) - ? 's3://' + process.env.ASSET_BUCKET_AUDIO + assetUtils.getGuardianAssetStoragePath('audio', dbRow.measured_at, dbRow.Guardian.guid, dbRow.Format.file_extension) + ? 's3://' + process.env.INGEST_BUCKET + audioPath : dbRow.url audioUtils.cacheSourceAudio(audioStorageUrl)