From e7f59e2e049903b3e4e200d8dce3f2a11c5e29a1 Mon Sep 17 00:00:00 2001 From: olzzon Date: Mon, 11 Mar 2024 11:14:54 +0100 Subject: [PATCH 1/7] wip: hotStandby - implement in device settings --- packages/mos-gateway/src/$schemas/devices.json | 6 ++++++ packages/mos-gateway/src/generated/devices.ts | 1 + 2 files changed, 7 insertions(+) diff --git a/packages/mos-gateway/src/$schemas/devices.json b/packages/mos-gateway/src/$schemas/devices.json index 1281fa38e5..1bf9b35c5d 100644 --- a/packages/mos-gateway/src/$schemas/devices.json +++ b/packages/mos-gateway/src/$schemas/devices.json @@ -105,6 +105,12 @@ "ui:description": "How often to ping NRCS to determine connection status", "default": 30000 }, + "hotStandby": { + "type": "boolean", + "ui:title": "Secondary: Hot Standby", + "ui:description": "Is the secondary connection a hot standby for the primary", + "default": false + }, "ports": { "type": "object", "ui:title": "Ports", diff --git a/packages/mos-gateway/src/generated/devices.ts b/packages/mos-gateway/src/generated/devices.ts index 4585db5efd..59bf8a31b2 100644 --- a/packages/mos-gateway/src/generated/devices.ts +++ b/packages/mos-gateway/src/generated/devices.ts @@ -24,6 +24,7 @@ export interface MosDeviceConfig { dontUseQueryPort?: boolean timeout?: number heartbeatInterval?: number + hotStandby?: boolean ports?: { lower: number upper: number From cef3236a1c5b741bb33abf4f13de2b616145c490 Mon Sep 17 00:00:00 2001 From: olzzon Date: Mon, 11 Mar 2024 11:19:29 +0100 Subject: [PATCH 2/7] wip: hotStandby - implement hotStandby error handling and messages --- .../mos-gateway/src/CoreMosDeviceHandler.ts | 42 +++++++++++++------ packages/mos-gateway/src/coreHandler.ts | 8 +++- packages/mos-gateway/src/mosHandler.ts | 9 ++-- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/packages/mos-gateway/src/CoreMosDeviceHandler.ts b/packages/mos-gateway/src/CoreMosDeviceHandler.ts index a5da2de82c..2eeefdd34c 100644 --- a/packages/mos-gateway/src/CoreMosDeviceHandler.ts +++ b/packages/mos-gateway/src/CoreMosDeviceHandler.ts @@ -75,13 +75,15 @@ export class CoreMosDeviceHandler { private _pendingStoryItemChanges: Array = [] private _pendingChangeTimeout: number = 60 * 1000 private mosTypes: MosTypes + private _hotStandby: boolean private _messageQueue: Queue - constructor(parent: CoreHandler, mosDevice: IMOSDevice, mosHandler: MosHandler) { + constructor(parent: CoreHandler, mosDevice: IMOSDevice, mosHandler: MosHandler, hotStandby: boolean) { this._coreParentHandler = parent this._mosDevice = mosDevice this._mosHandler = mosHandler + this._hotStandby = hotStandby this._messageQueue = new Queue() @@ -138,25 +140,39 @@ export class CoreMosDeviceHandler { let statusCode: StatusCode const messages: Array = [] - if (connectionStatus.PrimaryConnected) { - if (connectionStatus.SecondaryConnected || !this._mosDevice.idSecondary) { + if (this._hotStandby) { + if (connectionStatus.PrimaryConnected) { statusCode = StatusCode.GOOD } else { - statusCode = StatusCode.WARNING_MINOR + if (connectionStatus.SecondaryConnected) { + statusCode = StatusCode.GOOD + messages.push(connectionStatus.SecondaryStatus || 'Running NRCS on hot standby') + } else { + statusCode = StatusCode.BAD + messages.push(connectionStatus.SecondaryStatus || 'Primary and hot standby are not connected') + } } } else { - if (connectionStatus.SecondaryConnected) { - statusCode = StatusCode.WARNING_MAJOR + if (connectionStatus.PrimaryConnected) { + if (connectionStatus.SecondaryConnected || !this._mosDevice.idSecondary) { + statusCode = StatusCode.GOOD + } else { + statusCode = StatusCode.WARNING_MINOR + } } else { - statusCode = StatusCode.BAD + if (connectionStatus.SecondaryConnected) { + statusCode = StatusCode.WARNING_MAJOR + } else { + statusCode = StatusCode.BAD + } } - } - if (!connectionStatus.PrimaryConnected) { - messages.push(connectionStatus.PrimaryStatus || 'Primary not connected') - } - if (this._mosDevice.idSecondary && !connectionStatus.SecondaryConnected) { - messages.push(connectionStatus.SecondaryStatus || 'Fallback not connected') + if (!connectionStatus.PrimaryConnected) { + messages.push(connectionStatus.PrimaryStatus || 'Primary not connected') + } + if (this._mosDevice.idSecondary && !connectionStatus.SecondaryConnected) { + messages.push(connectionStatus.SecondaryStatus || 'Fallback not connected') + } } this.core diff --git a/packages/mos-gateway/src/coreHandler.ts b/packages/mos-gateway/src/coreHandler.ts index dee6f39bf6..502aa756c4 100644 --- a/packages/mos-gateway/src/coreHandler.ts +++ b/packages/mos-gateway/src/coreHandler.ts @@ -142,9 +142,13 @@ export class CoreHandler { return options } - async registerMosDevice(mosDevice: IMOSDevice, mosHandler: MosHandler): Promise { + async registerMosDevice( + mosDevice: IMOSDevice, + mosHandler: MosHandler, + hotStandby: boolean + ): Promise { this.logger.info('registerMosDevice -------------') - const coreMos = new CoreMosDeviceHandler(this, mosDevice, mosHandler) + const coreMos = new CoreMosDeviceHandler(this, mosDevice, mosHandler, hotStandby) this._coreMosHandlers.push(coreMos) return coreMos.init().then(() => { diff --git a/packages/mos-gateway/src/mosHandler.ts b/packages/mos-gateway/src/mosHandler.ts index c06429124d..365a674c67 100644 --- a/packages/mos-gateway/src/mosHandler.ts +++ b/packages/mos-gateway/src/mosHandler.ts @@ -59,6 +59,7 @@ export class MosHandler { private _logger: Winston.Logger private _disposed = false private _settings?: MosGatewayConfig + private _hotStandby: boolean private _coreHandler: CoreHandler | undefined private _observers: Array> = [] private _triggerupdateDevicesTimeout: any = null @@ -66,6 +67,7 @@ export class MosHandler { constructor(logger: Winston.Logger) { this._logger = logger + this._hotStandby = false this.mosTypes = getMosTypes(this.strict) // temporary, another will be set upon init() } async init(config: MosConfig, coreHandler: CoreHandler): Promise { @@ -101,7 +103,7 @@ export class MosHandler { this.mosTypes = getMosTypes(this.strict) - await this._initMosConnection() + await this._updateDevices() if (!this._coreHandler) throw Error('_coreHandler is undefined!') this._coreHandler.onConnected(() => { @@ -110,8 +112,6 @@ export class MosHandler { this.sendStatusOfAllMosDevices() }) this.setupObservers() - - return this._updateDevices() } async dispose(): Promise { this._disposed = true @@ -243,7 +243,7 @@ export class MosHandler { if (!this._coreHandler) throw Error('_coreHandler is undefined!') - const coreMosHandler = await this._coreHandler.registerMosDevice(mosDevice, this) + const coreMosHandler = await this._coreHandler.registerMosDevice(mosDevice, this, this._hotStandby) // this._logger.info('mosDevice registered -------------') // Setup message flow between the devices: @@ -420,6 +420,7 @@ export class MosHandler { for (const [deviceId, device] of Object.entries<{ options: MosDeviceConfig }>(devices)) { if (device) { if (device.options.secondary) { + this._hotStandby = device.options.secondary?.hotStandby || false // If the host isn't set, don't use secondary: if (!device.options.secondary.host || !device.options.secondary.id) delete device.options.secondary From 484b0d78e78ab1469986eec170a62d4428ebdc44 Mon Sep 17 00:00:00 2001 From: olzzon Date: Thu, 21 Mar 2024 11:42:17 +0100 Subject: [PATCH 3/7] wip: hotStandby - comment error message behavior --- packages/mos-gateway/src/CoreMosDeviceHandler.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/mos-gateway/src/CoreMosDeviceHandler.ts b/packages/mos-gateway/src/CoreMosDeviceHandler.ts index 2eeefdd34c..ed30017868 100644 --- a/packages/mos-gateway/src/CoreMosDeviceHandler.ts +++ b/packages/mos-gateway/src/CoreMosDeviceHandler.ts @@ -141,9 +141,12 @@ export class CoreMosDeviceHandler { const messages: Array = [] if (this._hotStandby) { + // OpenMedia treats secondary server as hot-standby + // And thus is not considered as a warning if it's not connected if (connectionStatus.PrimaryConnected) { statusCode = StatusCode.GOOD } else { + // Primary not connected is only bad if there is no secondary: if (connectionStatus.SecondaryConnected) { statusCode = StatusCode.GOOD messages.push(connectionStatus.SecondaryStatus || 'Running NRCS on hot standby') @@ -154,6 +157,7 @@ export class CoreMosDeviceHandler { } } else { if (connectionStatus.PrimaryConnected) { + // ENPS expect both Primary and Secondary to be connected if both of them are configured if (connectionStatus.SecondaryConnected || !this._mosDevice.idSecondary) { statusCode = StatusCode.GOOD } else { @@ -161,8 +165,10 @@ export class CoreMosDeviceHandler { } } else { if (connectionStatus.SecondaryConnected) { + // Primary not connected should give a warning if Secondary is used. statusCode = StatusCode.WARNING_MAJOR } else { + // If neither Primary nor Secondary is connected, it's a bad state. statusCode = StatusCode.BAD } } From 72c765f56f4acf991bbb566fefb2b99106536587 Mon Sep 17 00:00:00 2001 From: olzzon Date: Wed, 28 Aug 2024 12:33:58 +0200 Subject: [PATCH 4/7] fix: MosGateway hotstandby - send message for both servers if both connections are offline --- packages/mos-gateway/src/CoreMosDeviceHandler.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/mos-gateway/src/CoreMosDeviceHandler.ts b/packages/mos-gateway/src/CoreMosDeviceHandler.ts index 1bbcb4a8df..cd0d29d6a3 100644 --- a/packages/mos-gateway/src/CoreMosDeviceHandler.ts +++ b/packages/mos-gateway/src/CoreMosDeviceHandler.ts @@ -152,6 +152,8 @@ export class CoreMosDeviceHandler { messages.push(connectionStatus.SecondaryStatus || 'Running NRCS on hot standby') } else { statusCode = StatusCode.BAD + // Send messages for both connections + messages.push(connectionStatus.PrimaryStatus || 'Primary and hot standby are not connected') messages.push(connectionStatus.SecondaryStatus || 'Primary and hot standby are not connected') } } From ce6854336f4df8fc4f4b512a3de1f039fc0f1309 Mon Sep 17 00:00:00 2001 From: olzzon Date: Tue, 24 Sep 2024 13:37:14 +0200 Subject: [PATCH 5/7] fix: make openMediaHotStandby option pr subdevice, and rename from hotStandby to openMediaHotStandby --- .../mos-gateway/src/$schemas/devices.json | 40 +++++++++++++------ .../mos-gateway/src/CoreMosDeviceHandler.ts | 8 ++-- packages/mos-gateway/src/coreHandler.ts | 4 +- packages/mos-gateway/src/generated/devices.ts | 2 +- packages/mos-gateway/src/mosHandler.ts | 13 ++++-- 5 files changed, 44 insertions(+), 23 deletions(-) diff --git a/packages/mos-gateway/src/$schemas/devices.json b/packages/mos-gateway/src/$schemas/devices.json index 1bf9b35c5d..ae44a97199 100644 --- a/packages/mos-gateway/src/$schemas/devices.json +++ b/packages/mos-gateway/src/$schemas/devices.json @@ -61,13 +61,20 @@ "ui:title": "(Optional) MOS Query Port", "ui:description": "Connect to an alternate port for 'query' port MOS messages", "default": 10542 - } + } }, - "required": ["lower", "upper", "query"], + "required": [ + "lower", + "upper", + "query" + ], "additionalProperties": false } }, - "required": ["id", "host"], + "required": [ + "id", + "host" + ], "additionalProperties": false }, "secondary": { @@ -105,10 +112,10 @@ "ui:description": "How often to ping NRCS to determine connection status", "default": 30000 }, - "hotStandby": { + "openMediaHotStandby": { "type": "boolean", - "ui:title": "Secondary: Hot Standby", - "ui:description": "Is the secondary connection a hot standby for the primary", + "ui:title": "Secondary: OpenMedia Hot Standby", + "ui:description": "Is the secondary connection a OpenMedia hot standby for the primary", "default": false }, "ports": { @@ -132,16 +139,25 @@ "ui:title": "(Optional) MOS Query Port", "ui:description": "Connect to an alternate port for 'query' port MOS messages", "default": 10542 - } + } }, - "required": ["lower", "upper", "query"], + "required": [ + "lower", + "upper", + "query" + ], "additionalProperties": false - } + } }, - "required": ["id", "host"], + "required": [ + "id", + "host" + ], "additionalProperties": false } }, - "required": ["primary"], + "required": [ + "primary" + ], "additionalProperties": false -} +} \ No newline at end of file diff --git a/packages/mos-gateway/src/CoreMosDeviceHandler.ts b/packages/mos-gateway/src/CoreMosDeviceHandler.ts index cd0d29d6a3..d14598ffaa 100644 --- a/packages/mos-gateway/src/CoreMosDeviceHandler.ts +++ b/packages/mos-gateway/src/CoreMosDeviceHandler.ts @@ -75,15 +75,15 @@ export class CoreMosDeviceHandler { private _pendingStoryItemChanges: Array = [] private _pendingChangeTimeout: number = 60 * 1000 private mosTypes: MosTypes - private _hotStandby: boolean + private _openMediaHotStandby: boolean private _messageQueue: Queue - constructor(parent: CoreHandler, mosDevice: IMOSDevice, mosHandler: MosHandler, hotStandby: boolean) { + constructor(parent: CoreHandler, mosDevice: IMOSDevice, mosHandler: MosHandler, openMediaHotStandby: boolean) { this._coreParentHandler = parent this._mosDevice = mosDevice this._mosHandler = mosHandler - this._hotStandby = hotStandby + this._openMediaHotStandby = openMediaHotStandby this._messageQueue = new Queue() @@ -140,7 +140,7 @@ export class CoreMosDeviceHandler { let statusCode: StatusCode const messages: Array = [] - if (this._hotStandby) { + if (this._openMediaHotStandby) { // OpenMedia treats secondary server as hot-standby // And thus is not considered as a warning if it's not connected if (connectionStatus.PrimaryConnected) { diff --git a/packages/mos-gateway/src/coreHandler.ts b/packages/mos-gateway/src/coreHandler.ts index 502aa756c4..c693aee6af 100644 --- a/packages/mos-gateway/src/coreHandler.ts +++ b/packages/mos-gateway/src/coreHandler.ts @@ -145,10 +145,10 @@ export class CoreHandler { async registerMosDevice( mosDevice: IMOSDevice, mosHandler: MosHandler, - hotStandby: boolean + openMediaHotStandby: boolean ): Promise { this.logger.info('registerMosDevice -------------') - const coreMos = new CoreMosDeviceHandler(this, mosDevice, mosHandler, hotStandby) + const coreMos = new CoreMosDeviceHandler(this, mosDevice, mosHandler, openMediaHotStandby) this._coreMosHandlers.push(coreMos) return coreMos.init().then(() => { diff --git a/packages/mos-gateway/src/generated/devices.ts b/packages/mos-gateway/src/generated/devices.ts index 59bf8a31b2..f192cf7614 100644 --- a/packages/mos-gateway/src/generated/devices.ts +++ b/packages/mos-gateway/src/generated/devices.ts @@ -24,7 +24,7 @@ export interface MosDeviceConfig { dontUseQueryPort?: boolean timeout?: number heartbeatInterval?: number - hotStandby?: boolean + openMediaHotStandby?: boolean ports?: { lower: number upper: number diff --git a/packages/mos-gateway/src/mosHandler.ts b/packages/mos-gateway/src/mosHandler.ts index 365a674c67..975dada3e3 100644 --- a/packages/mos-gateway/src/mosHandler.ts +++ b/packages/mos-gateway/src/mosHandler.ts @@ -59,7 +59,7 @@ export class MosHandler { private _logger: Winston.Logger private _disposed = false private _settings?: MosGatewayConfig - private _hotStandby: boolean + private _openMediaHotStandby: Record private _coreHandler: CoreHandler | undefined private _observers: Array> = [] private _triggerupdateDevicesTimeout: any = null @@ -67,7 +67,7 @@ export class MosHandler { constructor(logger: Winston.Logger) { this._logger = logger - this._hotStandby = false + this._openMediaHotStandby = {} this.mosTypes = getMosTypes(this.strict) // temporary, another will be set upon init() } async init(config: MosConfig, coreHandler: CoreHandler): Promise { @@ -243,7 +243,11 @@ export class MosHandler { if (!this._coreHandler) throw Error('_coreHandler is undefined!') - const coreMosHandler = await this._coreHandler.registerMosDevice(mosDevice, this, this._hotStandby) + const coreMosHandler = await this._coreHandler.registerMosDevice( + mosDevice, + this, + mosDevice.idSecondary ? this._openMediaHotStandby[mosDevice.idSecondary] : false + ) // this._logger.info('mosDevice registered -------------') // Setup message flow between the devices: @@ -420,7 +424,8 @@ export class MosHandler { for (const [deviceId, device] of Object.entries<{ options: MosDeviceConfig }>(devices)) { if (device) { if (device.options.secondary) { - this._hotStandby = device.options.secondary?.hotStandby || false + this._openMediaHotStandby[device.options.secondary.id] = + device.options.secondary?.openMediaHotStandby || false // If the host isn't set, don't use secondary: if (!device.options.secondary.host || !device.options.secondary.id) delete device.options.secondary From e2fe4467df8b578661c74c5755c110f730f3c3d4 Mon Sep 17 00:00:00 2001 From: olzzon Date: Tue, 8 Oct 2024 11:40:50 +0200 Subject: [PATCH 6/7] fix: dis-/en-able heartbeat when openMediaHotStandby --- packages/mos-gateway/src/mosHandler.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/mos-gateway/src/mosHandler.ts b/packages/mos-gateway/src/mosHandler.ts index 975dada3e3..8fc2d08694 100644 --- a/packages/mos-gateway/src/mosHandler.ts +++ b/packages/mos-gateway/src/mosHandler.ts @@ -488,6 +488,11 @@ export class MosHandler { deviceOptions.primary.heartbeatInterval = deviceOptions.primary.heartbeatInterval || DEFAULT_MOS_HEARTBEAT_INTERVAL + if (deviceOptions.secondary?.id && this._openMediaHotStandby[deviceOptions.secondary.id]) { + //@ts-expect-error this is not yet added to the official mos-connection + deviceOptions.secondary.openMediaHotStandby = true + } + const mosDevice: MosDevice = await this.mos.connect(deviceOptions) this._ownMosDevices[deviceId] = mosDevice From 685862bdbcbb0ca1fe83372263b3dd1bdcc05ac6 Mon Sep 17 00:00:00 2001 From: Julian Waller Date: Tue, 10 Dec 2024 15:52:54 +0000 Subject: [PATCH 7/7] chore: review comments --- .../mos-gateway/src/CoreMosDeviceHandler.ts | 17 +++++++++++++---- packages/mos-gateway/src/coreHandler.ts | 6 +++--- packages/mos-gateway/src/mosHandler.ts | 10 +++++----- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/mos-gateway/src/CoreMosDeviceHandler.ts b/packages/mos-gateway/src/CoreMosDeviceHandler.ts index aafad18741..1ce92433dc 100644 --- a/packages/mos-gateway/src/CoreMosDeviceHandler.ts +++ b/packages/mos-gateway/src/CoreMosDeviceHandler.ts @@ -62,6 +62,10 @@ interface IStoryItemChange { itemDiff: PartialDeep } +export interface CoreMosDeviceHandlerOptions { + openMediaHotStandby?: boolean +} + /** * Represents a connection between a mos-device and Core */ @@ -76,15 +80,20 @@ export class CoreMosDeviceHandler { private _pendingStoryItemChanges: Array = [] private _pendingChangeTimeout: number = 60 * 1000 private mosTypes: MosTypes - private _openMediaHotStandby: boolean + private _options: CoreMosDeviceHandlerOptions private _messageQueue: Queue - constructor(parent: CoreHandler, mosDevice: IMOSDevice, mosHandler: MosHandler, openMediaHotStandby: boolean) { + constructor( + parent: CoreHandler, + mosDevice: IMOSDevice, + mosHandler: MosHandler, + options: CoreMosDeviceHandlerOptions + ) { this._coreParentHandler = parent this._mosDevice = mosDevice this._mosHandler = mosHandler - this._openMediaHotStandby = openMediaHotStandby + this._options = options this._messageQueue = new Queue() @@ -141,7 +150,7 @@ export class CoreMosDeviceHandler { let statusCode: StatusCode const messages: Array = [] - if (this._openMediaHotStandby) { + if (this._options.openMediaHotStandby) { // OpenMedia treats secondary server as hot-standby // And thus is not considered as a warning if it's not connected if (connectionStatus.PrimaryConnected) { diff --git a/packages/mos-gateway/src/coreHandler.ts b/packages/mos-gateway/src/coreHandler.ts index d943a49383..2f578fb61a 100644 --- a/packages/mos-gateway/src/coreHandler.ts +++ b/packages/mos-gateway/src/coreHandler.ts @@ -18,7 +18,7 @@ import { MosHandler } from './mosHandler' import { DeviceConfig } from './connector' import { MOS_DEVICE_CONFIG_MANIFEST } from './configManifest' import { getVersions } from './versions' -import { CoreMosDeviceHandler } from './CoreMosDeviceHandler' +import { CoreMosDeviceHandler, CoreMosDeviceHandlerOptions } from './CoreMosDeviceHandler' import { PeripheralDeviceCommandId } from '@sofie-automation/shared-lib/dist/core/model/Ids' export interface CoreConfig { @@ -145,10 +145,10 @@ export class CoreHandler { async registerMosDevice( mosDevice: IMOSDevice, mosHandler: MosHandler, - openMediaHotStandby: boolean + deviceOptions: CoreMosDeviceHandlerOptions ): Promise { this.logger.info('registerMosDevice -------------') - const coreMos = new CoreMosDeviceHandler(this, mosDevice, mosHandler, openMediaHotStandby) + const coreMos = new CoreMosDeviceHandler(this, mosDevice, mosHandler, deviceOptions) this._coreMosHandlers.push(coreMos) return coreMos.init().then(() => { diff --git a/packages/mos-gateway/src/mosHandler.ts b/packages/mos-gateway/src/mosHandler.ts index 8fc2d08694..572238dc95 100644 --- a/packages/mos-gateway/src/mosHandler.ts +++ b/packages/mos-gateway/src/mosHandler.ts @@ -243,11 +243,11 @@ export class MosHandler { if (!this._coreHandler) throw Error('_coreHandler is undefined!') - const coreMosHandler = await this._coreHandler.registerMosDevice( - mosDevice, - this, - mosDevice.idSecondary ? this._openMediaHotStandby[mosDevice.idSecondary] : false - ) + const coreMosHandler = await this._coreHandler.registerMosDevice(mosDevice, this, { + openMediaHotStandby: mosDevice.idSecondary + ? this._openMediaHotStandby[mosDevice.idSecondary] + : false, + }) // this._logger.info('mosDevice registered -------------') // Setup message flow between the devices: