wip: rework piece expected packages SOFIE-3402

meteor/lib/collections/ExpectedPackages.ts

@@ -6,9 +6,10 @@ import {
 	htmlTemplateGetFileNamesFromSteps,
 } from '@sofie-automation/shared-lib/dist/package-manager/helpers'
 import deepExtend from 'deep-extend'
+import { ReadonlyDeep } from 'type-fest'
 
 export function getPreviewPackageSettings(
-	expectedPackage: ExpectedPackage.Any
+	expectedPackage: ReadonlyDeep<ExpectedPackage.Any>
 ): ExpectedPackage.SideEffectPreviewSettings | undefined {
 	if (expectedPackage.type === ExpectedPackage.PackageType.MEDIA_FILE) {
 		const packagePath = expectedPackage.content.filePath
@@ -29,7 +30,7 @@ export function getPreviewPackageSettings(
 	}
 }
 export function getThumbnailPackageSettings(
-	expectedPackage: ExpectedPackage.Any
+	expectedPackage: ReadonlyDeep<ExpectedPackage.Any>
 ): ExpectedPackage.SideEffectThumbnailSettings | undefined {
 	if (expectedPackage.type === ExpectedPackage.PackageType.MEDIA_FILE) {
 		const packagePath = expectedPackage.content.filePath
@@ -51,16 +52,16 @@ export function getThumbnailPackageSettings(
 	}
 }
 export function getSideEffect(
-	expectedPackage: ExpectedPackage.Base,
+	expectedPackage: ReadonlyDeep<ExpectedPackage.Base>,
 	studio: Pick<StudioLight, 'previewContainerIds' | 'thumbnailContainerIds'>
 ): ExpectedPackage.Base['sideEffect'] {
 	return deepExtend(
 		{},
 		literal<ExpectedPackage.Base['sideEffect']>({
 			previewContainerId: studio.previewContainerIds[0], // just pick the first. Todo: something else?
 			thumbnailContainerId: studio.thumbnailContainerIds[0], // just pick the first. Todo: something else?
-			previewPackageSettings: getPreviewPackageSettings(expectedPackage as ExpectedPackage.Any),
-			thumbnailPackageSettings: getThumbnailPackageSettings(expectedPackage as ExpectedPackage.Any),
+			previewPackageSettings: getPreviewPackageSettings(expectedPackage as ReadonlyDeep<ExpectedPackage.Any>),
+			thumbnailPackageSettings: getThumbnailPackageSettings(expectedPackage as ReadonlyDeep<ExpectedPackage.Any>),
 		}),
 		expectedPackage.sideEffect
 	)

meteor/server/api/__tests__/cleanup.test.ts

@@ -256,19 +256,14 @@ async function setDefaultDatatoDB(env: DefaultEnvironment, now: number) {
 		_id: getRandomId(),
 		blueprintPackageId: '',
 		bucketId,
-		content: {} as any,
+		package: {} as any,
 		contentVersionHash: '',
 		created: 0,
 		fromPieceType: '' as any,
-		layers: [],
 		pieceId,
 		rundownId,
 		segmentId,
-		sideEffect: {} as any,
 		studioId,
-		sources: {} as any,
-		type: '' as any,
-		version: {} as any,
 	})
 	await ExpectedPackageWorkStatuses.insertAsync({
 		_id: getRandomId(),

meteor/server/api/ingest/packageInfo.ts

@@ -27,7 +27,7 @@ export async function onUpdatedPackageInfo(packageId: ExpectedPackageId, _doc: P
 		return
 	}
 
-	if (pkg.listenToPackageInfoUpdates) {
+	if (pkg.package.listenToPackageInfoUpdates) {
 		switch (pkg.fromPieceType) {
 			case ExpectedPackageDBType.PIECE:
 			case ExpectedPackageDBType.ADLIB_PIECE:
@@ -44,6 +44,9 @@ export async function onUpdatedPackageInfo(packageId: ExpectedPackageId, _doc: P
 			case ExpectedPackageDBType.STUDIO_BASELINE_OBJECTS:
 				onUpdatedPackageInfoForStudioBaselineDebounce(pkg)
 				break
+			case ExpectedPackageDBType.PIECE_INSTANCE:
+				// No-op, we can't handle these updates
+				break
 			default:
 				assertNever(pkg)
 				break

meteor/server/publications/packageManager/expectedPackages/generate.ts

@@ -47,7 +47,7 @@ export async function updateCollectionForExpectedPackageIds(
 
 		// Map the expectedPackages onto their specified layer:
 		const allDeviceIds = new Set<PeripheralDeviceId>()
-		for (const layerName of packageDoc.layers) {
+		for (const layerName of packageDoc.package.layers) {
 			const layerDeviceIds = layerNameToDeviceIds.get(layerName)
 			for (const deviceId of layerDeviceIds || []) {
 				allDeviceIds.add(deviceId)
@@ -61,8 +61,9 @@ export async function updateCollectionForExpectedPackageIds(
 			const routedPackage = generateExpectedPackageForDevice(
 				studio,
 				{
-					...packageDoc,
+					...packageDoc.package,
 					_id: unprotectString(packageDoc._id),
+					rundownId: 'rundownId' in packageDoc ? packageDoc.rundownId : undefined,
 				},
 				deviceId,
 				null,
@@ -207,11 +208,13 @@ function generateExpectedPackageForDevice(
 	if (!combinedTargets.length) {
 		logger.warn(`Pub.expectedPackagesForDevice: No targets found for "${expectedPackage._id}"`)
 	}
-	expectedPackage.sideEffect = getSideEffect(expectedPackage, studio)
 
 	return {
 		_id: protectString(`${expectedPackage._id}_${deviceId}_${pieceInstanceId}`),
-		expectedPackage: expectedPackage,
+		expectedPackage: {
+			...expectedPackage,
+			sideEffect: getSideEffect(expectedPackage, studio),
+		},
 		sources: combinedSources,
 		targets: combinedTargets,
 		priority: priority,
@@ -239,7 +242,7 @@ function calculateCombinedSource(
 	for (const accessorId of accessorIds) {
 		const sourceAccessor: Accessor.Any | undefined = lookedUpSource.container.accessors[accessorId]
 
-		const packageAccessor: AccessorOnPackage.Any | undefined = packageSource.accessors?.[accessorId]
+		const packageAccessor: ReadonlyDeep<AccessorOnPackage.Any> | undefined = packageSource.accessors?.[accessorId]
 
 		if (packageAccessor && sourceAccessor && packageAccessor.type === sourceAccessor.type) {
 			combinedSource.accessors[accessorId] = deepExtend({}, sourceAccessor, packageAccessor)

packages/corelib/src/dataModel/AdLibPiece.ts

@@ -2,7 +2,7 @@ import { IBlueprintAdLibPiece, SomeContent } from '@sofie-automation/blueprints-
 import { RundownId, PartId } from './Ids'
 import { PieceGeneric } from './Piece'
 
-export interface AdLibPiece extends PieceGeneric, Omit<IBlueprintAdLibPiece, 'content'> {
+export interface AdLibPiece extends PieceGeneric, Omit<IBlueprintAdLibPiece, 'content' | 'expectedPackages'> {
 	/** Rundown this AdLib belongs to */
 	rundownId: RundownId
 

packages/corelib/src/dataModel/AdlibAction.ts

@@ -1,13 +1,13 @@
 import { IBlueprintActionManifest } from '@sofie-automation/blueprints-integration'
 import { ArrayElement } from '../lib'
 import { ITranslatableMessage } from '../TranslatableMessage'
-import { ProtectedStringProperties } from '../protectedString'
 import { RundownId, AdLibActionId, PartId } from './Ids'
+import { PieceExpectedPackage } from './Piece'
 
 /** The following extended interface allows assigning namespace information to the actions as they are stored in the
  *  database after being emitted from the blueprints
  */
-export interface AdLibActionCommon extends ProtectedStringProperties<IBlueprintActionManifest, 'partId'> {
+export interface AdLibActionCommon extends Omit<IBlueprintActionManifest, 'partId' | 'expectedPackages'> {
 	rundownId: RundownId
 	display: IBlueprintActionManifest['display'] & {
 		// this property can be a string if the name is modified by the User
@@ -22,6 +22,8 @@ export interface AdLibActionCommon extends ProtectedStringProperties<IBlueprintA
 		}
 	})[]
 
+	expectedPackages: PieceExpectedPackage[]
+
 	/**
 	 * String that can be used to identify adlibs that are equivalent to each other,
 	 * if there are multiple Adlibs with the same uniquenessId,

packages/corelib/src/dataModel/BucketAdLibAction.ts

@@ -2,11 +2,15 @@ import { BucketAdLibActionId, BucketId, StudioId, ShowStyleVariantId, ShowStyleB
 import { RundownImportVersions } from './Rundown'
 import { AdLibActionCommon } from './AdlibAction'
 import { BucketAdLibIngestInfo } from './BucketAdLibPiece'
+import { IBlueprintActionManifest } from '@sofie-automation/blueprints-integration'
 
-export interface BucketAdLibAction extends Omit<AdLibActionCommon, 'rundownId'> {
+export interface BucketAdLibAction extends Omit<AdLibActionCommon, 'rundownId' | 'expectedPackages'> {
 	_id: BucketAdLibActionId
 	bucketId: BucketId
 
+	// nocommit - temporary copy to avoid type errors
+	expectedPackages: IBlueprintActionManifest['expectedPackages']
+
 	externalId: string
 
 	/**

packages/corelib/src/dataModel/ExpectedPackages.ts

@@ -5,7 +5,6 @@ import {
 	AdLibActionId,
 	BucketAdLibActionId,
 	BucketAdLibId,
-	BucketId,
 	ExpectedPackageId,
 	PartId,
 	PieceId,
@@ -26,20 +25,13 @@ import { ReadonlyDeep } from 'type-fest'
    The Package Manager will then copy the file to the right place.
 */
 
-export type ExpectedPackageFromRundown = ExpectedPackageDBFromPiece | ExpectedPackageDBFromAdLibAction
-
-export type ExpectedPackageFromRundownBaseline =
-	| ExpectedPackageDBFromBaselineAdLibAction
-	| ExpectedPackageDBFromBaselineAdLibPiece
-	| ExpectedPackageDBFromRundownBaselineObjects
-
 export type ExpectedPackageDBFromBucket = ExpectedPackageDBFromBucketAdLib | ExpectedPackageDBFromBucketAdLibAction
 
-export type ExpectedPackageDB =
-	| ExpectedPackageFromRundown
-	| ExpectedPackageDBFromBucket
-	| ExpectedPackageFromRundownBaseline
-	| ExpectedPackageDBFromStudioBaselineObjects
+// export type ExpectedPackageDB =
+// 	| ExpectedPackageFromRundown
+// 	| ExpectedPackageDBFromBucket
+// 	| ExpectedPackageFromRundownBaseline
+// 	| ExpectedPackageDBFromStudioBaselineObjects
 
 export enum ExpectedPackageDBType {
 	PIECE = 'piece',
@@ -51,89 +43,111 @@ export enum ExpectedPackageDBType {
 	BUCKET_ADLIB_ACTION = 'bucket_adlib_action',
 	RUNDOWN_BASELINE_OBJECTS = 'rundown_baseline_objects',
 	STUDIO_BASELINE_OBJECTS = 'studio_baseline_objects',
+	PIECE_INSTANCE = 'piece_instance',
 }
-export interface ExpectedPackageDBBase extends Omit<ExpectedPackage.Base, '_id'> {
-	_id: ExpectedPackageId
-	/** The local package id - as given by the blueprints */
-	blueprintPackageId: string
+
+/*
+ * What about this new concept. The aim here is to avoid the constant inserting and deleting of expectedPackages during playout, and avoiding duplicate packages with the same content.
+ * The idea is to have a single expectedPackage for each 'content'.
+ * Ingest will 'deduplicate' the packages produced by the blueprints, with playout able to reference them with pieceInstanceIds.
+ *
+ * During the ingest save phase, it will need to reload the `playoutSources` property, in case it has changed. And if there are uses remaining, it will need to keep the package after clearing the `ingestSources`.
+ * During playout operations, pieceInstanceIds will be added and removed as needed. If there remains no sources (of either type), then the document can be removed. If an in-progress ingest tried to reclaim it, it will get reinserted.
+ *
+ * Playout can then load just the ones referenced by piece instances, and just before it needs to use them (for bluerpint types or something), can ensure that everything needed has been loaded.
+ * During a take, any packages referenced by the previous(?) partinstance must be removed.
+ * When doing a reset of the rundown, all playout references must be removed.
+ * When inserting/removing pieceinstances, the expectedPackages must be updated.
+ */
+export interface ExpectedPackageDBNew {
+	_id: ExpectedPackageId // derived from rundownId and hash of `package`
+
+	// /** The local package id - as given by the blueprints */
+	// blueprintPackageId: string // TODO - remove this?
 
 	/** The studio of the Rundown of the Piece this package belongs to */
 	studioId: StudioId
 
+	/** The rundown of the Piece this package belongs to */
+	rundownId: RundownId
+
 	/** Hash that changes whenever the content or version changes. See getContentVersionHash() */
 	contentVersionHash: string
 
-	// pieceId: ProtectedString<any> | null
-	fromPieceType: ExpectedPackageDBType
-
 	created: Time
+
+	package: ReadonlyDeep<ExpectedPackage.Any>
+
+	ingestSources: ExpectedPackageIngestSource[]
+
+	playoutSources: {
+		/** Any playout PieceInstance. This is limited to the current and next partInstances */ // nocommit - verify this
+		pieceInstanceIds: PieceInstanceId[]
+	}
 }
-export interface ExpectedPackageDBFromPiece extends ExpectedPackageDBBase {
+
+export interface ExpectedPackageIngestSourcePiece {
 	fromPieceType: ExpectedPackageDBType.PIECE | ExpectedPackageDBType.ADLIB_PIECE
 	/** The Piece this package belongs to */
 	pieceId: PieceId
 	/** The Part this package belongs to */
 	partId: PartId
 	/** The Segment this package belongs to */
 	segmentId: SegmentId
-	/** The rundown of the Piece this package belongs to */
-	rundownId: RundownId
-}
-
-export interface ExpectedPackageDBFromBaselineAdLibPiece extends ExpectedPackageDBBase {
-	fromPieceType: ExpectedPackageDBType.BASELINE_ADLIB_PIECE
-	/** The Piece this package belongs to */
-	pieceId: PieceId
-	/** The rundown of the Piece this package belongs to */
-	rundownId: RundownId
 }
-
-export interface ExpectedPackageDBFromAdLibAction extends ExpectedPackageDBBase {
+export interface ExpectedPackageIngestSourceAdlibAction {
 	fromPieceType: ExpectedPackageDBType.ADLIB_ACTION
-	/** The Adlib Action this package belongs to */
+	/** The Piece this package belongs to */
 	pieceId: AdLibActionId
 	/** The Part this package belongs to */
 	partId: PartId
 	/** The Segment this package belongs to */
 	segmentId: SegmentId
-	/** The rundown of the Piece this package belongs to */
-	rundownId: RundownId
 }
-export interface ExpectedPackageDBFromBaselineAdLibAction extends ExpectedPackageDBBase {
+export interface ExpectedPackageIngestSourceBaselineAdlibPiece {
+	fromPieceType: ExpectedPackageDBType.BASELINE_ADLIB_PIECE
+	/** The Piece this package belongs to */
+	pieceId: PieceId
+}
+export interface ExpectedPackageIngestSourceBaselineAdlibAction {
 	fromPieceType: ExpectedPackageDBType.BASELINE_ADLIB_ACTION
 	/** The Piece this package belongs to */
 	pieceId: RundownBaselineAdLibActionId
-	/** The rundown of the Piece this package belongs to */
-	rundownId: RundownId
 }
-
-export interface ExpectedPackageDBFromRundownBaselineObjects extends ExpectedPackageDBBase {
+export interface ExpectedPackageIngestSourceBaselineObjects {
 	fromPieceType: ExpectedPackageDBType.RUNDOWN_BASELINE_OBJECTS
-	/** The rundown of the Piece this package belongs to */
-	rundownId: RundownId
-	pieceId: null
-}
-export interface ExpectedPackageDBFromStudioBaselineObjects extends ExpectedPackageDBBase {
-	fromPieceType: ExpectedPackageDBType.STUDIO_BASELINE_OBJECTS
-	pieceId: null
 }
 
-export interface ExpectedPackageDBFromBucketAdLib extends ExpectedPackageDBBase {
-	fromPieceType: ExpectedPackageDBType.BUCKET_ADLIB
-	bucketId: BucketId
-	/** The Bucket adlib this package belongs to */
-	pieceId: BucketAdLibId
-	/** The `externalId` of the Bucket adlib this package belongs to */
-	pieceExternalId: string
-}
-export interface ExpectedPackageDBFromBucketAdLibAction extends ExpectedPackageDBBase {
-	fromPieceType: ExpectedPackageDBType.BUCKET_ADLIB_ACTION
-	bucketId: BucketId
-	/** The Bucket adlib-action this package belongs to */
-	pieceId: BucketAdLibActionId
-	/** The `externalId` of the Bucket adlib-action this package belongs to */
-	pieceExternalId: string
-}
+export type ExpectedPackageIngestSourcePart = ExpectedPackageIngestSourcePiece | ExpectedPackageIngestSourceAdlibAction
+
+export type ExpectedPackageIngestSourceRundownBaseline =
+	| ExpectedPackageIngestSourceBaselineAdlibPiece
+	| ExpectedPackageIngestSourceBaselineAdlibAction
+	| ExpectedPackageIngestSourceBaselineObjects
+
+export type ExpectedPackageIngestSource = ExpectedPackageIngestSourcePart | ExpectedPackageIngestSourceRundownBaseline
+
+// export interface ExpectedPackageDBFromStudioBaselineObjects extends ExpectedPackageDBBase {
+// 	fromPieceType: ExpectedPackageDBType.STUDIO_BASELINE_OBJECTS
+// 	pieceId: null
+// }
+
+// export interface ExpectedPackageDBFromBucketAdLib extends ExpectedPackageDBBase {
+// 	fromPieceType: ExpectedPackageDBType.BUCKET_ADLIB
+// 	bucketId: BucketId
+// 	/** The Bucket adlib this package belongs to */
+// 	pieceId: BucketAdLibId
+// 	/** The `externalId` of the Bucket adlib this package belongs to */
+// 	pieceExternalId: string
+// }
+// export interface ExpectedPackageDBFromBucketAdLibAction extends ExpectedPackageDBBase {
+// 	fromPieceType: ExpectedPackageDBType.BUCKET_ADLIB_ACTION
+// 	bucketId: BucketId
+// 	/** The Bucket adlib-action this package belongs to */
+// 	pieceId: BucketAdLibActionId
+// 	/** The `externalId` of the Bucket adlib-action this package belongs to */
+// 	pieceExternalId: string
+// }
 
 export function getContentVersionHash(expectedPackage: ReadonlyDeep<Omit<ExpectedPackage.Any, '_id'>>): string {
 	return hashObj({
@@ -159,3 +173,18 @@ export function getExpectedPackageId(
 ): ExpectedPackageId {
 	return protectString(`${ownerId}_${getHash(localExpectedPackageId)}`)
 }
+
+export function getExpectedPackageIdNew(
+	/** _id of the rundown*/
+	rundownId: RundownId,
+	/** The locally unique id of the expectedPackage */
+	expectedPackage: ReadonlyDeep<ExpectedPackage.Any>
+): ExpectedPackageId {
+	// This may be too agressive, but we don't know how to merge some of the properties
+	const objHash = hashObj({
+		...expectedPackage,
+		listenToPackageInfoUpdates: false, // Not relevant for the hash
+	} satisfies ReadonlyDeep<ExpectedPackage.Any>)
+
+	return protectString(`${rundownId}_${getHash(objHash)}`)
+}