From eb9872714641d9640f57f70ef1534cf16a3408c2 Mon Sep 17 00:00:00 2001 From: Marco Link Date: Tue, 10 Oct 2023 23:07:51 +0200 Subject: [PATCH] feat: add force flag to import omitted fields [#62] --- lib/index.ts | 58 +++++++++++-------- .../force-delete-omitted-field-transform.ts | 9 +++ lib/transform/transform-space.ts | 8 ++- lib/transform/transformers.ts | 36 ++++++++---- lib/usageParams.ts | 5 ++ ...rce-delete-omitted-field-transform.test.ts | 11 ++++ test/unit/transform/transform-space.test.ts | 2 + test/unit/transform/transformers.test.ts | 6 +- 8 files changed, 94 insertions(+), 41 deletions(-) create mode 100644 lib/transform/force-delete-omitted-field-transform.ts create mode 100644 test/unit/transform/force-delete-omitted-field-transform.test.ts diff --git a/lib/index.ts b/lib/index.ts index a82474a3..5b095057 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -6,7 +6,7 @@ import UpdateRenderer from 'listr-update-renderer' import VerboseRenderer from 'listr-verbose-renderer' import { startCase } from 'lodash' import PQueue from 'p-queue' - +import { pipe } from 'lodash/fp' import { displayErrorLog, setupLogging, writeErrorLogFile } from 'contentful-batch-libs/dist/logging' import { wrapTask } from 'contentful-batch-libs/dist/listr' @@ -17,6 +17,9 @@ import transformSpace from './transform/transform-space' import { assertDefaultLocale, assertPayload } from './utils/validations' import parseOptions from './parseOptions' import { ContentfulMultiError, LogItem } from './utils/errors' +import { transformers } from './transform/transformers' +import { ContentTypeProps } from 'contentful-management' +import { forceDeleteOmittedFieldTransform } from './transform/force-delete-omitted-field-transform' const ONE_SECOND = 1000 @@ -39,28 +42,29 @@ function createListrOptions (options) { // These type definitions follow what is specified in the Readme type RunContentfulImportParams = { - spaceId: string, - environmentId?: string, - managementToken: string, - contentFile?: string, - content?: object, - contentModelOnly?: boolean, - skipContentModel?: boolean, - skipLocales?: boolean, - skipContentPublishing?: boolean, - uploadAssets?: boolean, - assetsDirectory?: string, - host?: string, - proxy?: string, - rawProxy?: string, - rateLimit?: number, - headers?: object, - errorLogFile?: string, - useVerboseRenderer?: boolean, - // TODO These properties are not documented in the Readme - timeout?: number, - retryLimit?: number, - config?: string, + spaceId: string, + environmentId?: string, + managementToken: string, + contentFile?: string, + content?: object, + contentModelOnly?: boolean, + skipContentModel?: boolean, + skipLocales?: boolean, + skipContentPublishing?: boolean, + uploadAssets?: boolean, + assetsDirectory?: string, + host?: string, + proxy?: string, + rawProxy?: string, + rateLimit?: number, + headers?: object, + errorLogFile?: string, + useVerboseRenderer?: boolean, + // TODO These properties are not documented in the Readme + timeout?: number, + retryLimit?: number, + config?: string, + force?: boolean, } async function runContentfulImport (params: RunContentfulImportParams) { @@ -133,8 +137,12 @@ async function runContentfulImport (params: RunContentfulImportParams) { { title: 'Apply transformations to source data', task: wrapTask(async (ctx) => { - const transformedSourceData = transformSpace(ctx.sourceDataUntransformed, ctx.destinationData) - ctx.sourceData = transformedSourceData + const customTransformers: Partial = {} + if (options.force) { + customTransformers.contentTypes = (contentType: ContentTypeProps) => pipe(transformers.contentTypes, forceDeleteOmittedFieldTransform)(contentType) + } + + ctx.sourceData = transformSpace(ctx.sourceDataUntransformed, ctx.destinationData, customTransformers) }) }, { diff --git a/lib/transform/force-delete-omitted-field-transform.ts b/lib/transform/force-delete-omitted-field-transform.ts new file mode 100644 index 00000000..a8abffde --- /dev/null +++ b/lib/transform/force-delete-omitted-field-transform.ts @@ -0,0 +1,9 @@ +import { ContentTypeProps } from 'contentful-management' + +export function forceDeleteOmittedFieldTransform (contentType: ContentTypeProps) { + const omittedFields = contentType.fields.filter(field => field.omitted) + omittedFields.forEach(field => { + contentType.fields = contentType.fields.filter(f => f.id !== field.id) + }) + return contentType +} diff --git a/lib/transform/transform-space.ts b/lib/transform/transform-space.ts index 3e588fc3..cbf2b007 100644 --- a/lib/transform/transform-space.ts +++ b/lib/transform/transform-space.ts @@ -1,6 +1,6 @@ -import { omit, defaults } from 'lodash/object' +import { omit, defaults } from 'lodash' -import * as defaultTransformers from './transformers' +import { transformers as defaultTransformers } from './transformers' import sortEntries from '../utils/sort-entries' import sortLocales from '../utils/sort-locales' import { DestinationData, OriginalSourceData, TransformedSourceData } from '../types' @@ -14,7 +14,7 @@ const spaceEntities = [ * is a need to transform data when copying it to the destination space */ export default function ( - sourceData: OriginalSourceData, destinationData: DestinationData, customTransformers?: any, entities = spaceEntities + sourceData: OriginalSourceData, destinationData: DestinationData, customTransformers?: Partial, entities = spaceEntities ): TransformedSourceData { const transformers = defaults(customTransformers, defaultTransformers) const baseSpaceData = omit(sourceData, ...entities) @@ -22,6 +22,8 @@ export default function ( sourceData.locales = sortLocales(sourceData.locales) const tagsEnabled = !!destinationData.tags + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore return entities.reduce((transformedSpaceData, type) => { // tags don't contain links to other entities, don't need to be sorted const sortedEntities = (type === 'tags') ? sourceData[type] : sortEntries(sourceData[type]) diff --git a/lib/transform/transformers.ts b/lib/transform/transformers.ts index 8ab2e78f..225a9d22 100644 --- a/lib/transform/transformers.ts +++ b/lib/transform/transformers.ts @@ -1,5 +1,6 @@ -import { ContentTypeProps, EntryProps, TagProps, WebhookProps } from 'contentful-management' -import { find, omit, pick, reduce } from 'lodash' +import { omit, pick, find, reduce } from 'lodash' +import { AssetProps, ContentTypeProps, EntryProps, LocaleProps, TagProps, WebhookProps } from 'contentful-management' +import { MetadataProps } from 'contentful-management/dist/typings/common-types' /** * Default transformer methods for each kind of entity. @@ -8,19 +9,19 @@ import { find, omit, pick, reduce } from 'lodash' * as the whole upload process needs to be followed again. */ -export function contentTypes (contentType: ContentTypeProps) { +function contentTypes (contentType: ContentTypeProps) { return contentType } -export function tags (tag: TagProps) { +function tags (tag: TagProps) { return tag } -export function entries (entry: EntryProps, _, tagsEnabled = false) { +function entries (entry: EntryProps, _, tagsEnabled = false) { return removeMetadataTags(entry, tagsEnabled) } -export function webhooks (webhook: WebhookProps) { +function webhooks (webhook: WebhookProps) { // Workaround for webhooks with credentials if (webhook.httpBasicUsername) { delete webhook.httpBasicUsername @@ -34,9 +35,13 @@ export function webhooks (webhook: WebhookProps) { return webhook } -export function assets (asset, _, tagsEnabled = false) { +function assets (asset: AssetProps, _, tagsEnabled = false) { const transformedAsset = omit(asset, 'sys') + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore transformedAsset.sys = pick(asset.sys, 'id') + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore transformedAsset.fields = pick(asset.fields, 'title', 'description') transformedAsset.fields.file = reduce( asset.fields.file, @@ -44,7 +49,7 @@ export function assets (asset, _, tagsEnabled = false) { newFile[locale] = pick(localizedFile, 'contentType', 'fileName') if (!localizedFile.uploadFrom) { const assetUrl = localizedFile.url || localizedFile.upload - newFile[locale].upload = `${/^(http|https):\/\//i.test(assetUrl) ? '' : 'https:'}${assetUrl}` + newFile[locale].upload = `${/^(http|https):\/\//i.test(assetUrl!) ? '' : 'https:'}${assetUrl}` } else { newFile[locale].uploadFrom = localizedFile.uploadFrom } @@ -55,7 +60,7 @@ export function assets (asset, _, tagsEnabled = false) { return removeMetadataTags(transformedAsset, tagsEnabled) } -export function locales (locale, destinationLocales) { +function locales (locale: LocaleProps, destinationLocales: Array): LocaleProps { const transformedLocale = pick(locale, 'code', 'name', 'contentManagementApi', 'contentDeliveryApi', 'fallbackCode', 'optional') const destinationLocale = find(destinationLocales, { code: locale.code }) if (destinationLocale) { @@ -66,12 +71,23 @@ export function locales (locale, destinationLocales) { transformedLocale.sys = pick(destinationLocale.sys, 'id') } + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore return transformedLocale } -function removeMetadataTags (entity, tagsEnabled = false) { +function removeMetadataTags (entity: T, tagsEnabled = false): T { if (!tagsEnabled) { delete entity.metadata } return entity } + +export const transformers = { + contentTypes, + tags, + entries, + webhooks, + assets, + locales +} diff --git a/lib/usageParams.ts b/lib/usageParams.ts index eb118db1..4298c300 100644 --- a/lib/usageParams.ts +++ b/lib/usageParams.ts @@ -84,5 +84,10 @@ export default yargs type: 'string', describe: 'Pass an additional HTTP Header' }) + .option('force', { + describe: 'force omitted fields to be deleted before importing', + type: 'boolean', + default: false + }) .config('config', 'An optional configuration JSON file containing all the options for a single run') .argv diff --git a/test/unit/transform/force-delete-omitted-field-transform.test.ts b/test/unit/transform/force-delete-omitted-field-transform.test.ts new file mode 100644 index 00000000..a4983e8b --- /dev/null +++ b/test/unit/transform/force-delete-omitted-field-transform.test.ts @@ -0,0 +1,11 @@ +import { cloneMock } from 'contentful-batch-libs/test/mocks/' +import { ContentTypeProps } from 'contentful-management' +import { forceDeleteOmittedFieldTransform } from '../../../lib/transform/force-delete-omitted-field-transform' + +test('It should transform content types by dropping omitted fields', () => { + const contentTypeMock = cloneMock('contentType') as ContentTypeProps + contentTypeMock.fields[0].omitted = true + expect(contentTypeMock.fields).toHaveLength(1) + const transformedContentTypeMock = forceDeleteOmittedFieldTransform(contentTypeMock) + expect(transformedContentTypeMock.fields).toHaveLength(0) +}) diff --git a/test/unit/transform/transform-space.test.ts b/test/unit/transform/transform-space.test.ts index ba187f8f..a2fa5585 100644 --- a/test/unit/transform/transform-space.test.ts +++ b/test/unit/transform/transform-space.test.ts @@ -60,6 +60,8 @@ test('applies transformers to give space data', () => { test('applies custom transformers to give space data', () => { const result = transformSpace(space, destinationSpace, { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore entries: () => 'transformed' }) expect(result.entries?.[0]?.transformed).toBe('transformed') diff --git a/test/unit/transform/transformers.test.ts b/test/unit/transform/transformers.test.ts index 5d74ce75..71f6d988 100644 --- a/test/unit/transform/transformers.test.ts +++ b/test/unit/transform/transformers.test.ts @@ -1,6 +1,6 @@ import { cloneMock } from 'contentful-batch-libs/test/mocks/' -import * as transformers from '../../../lib/transform/transformers' +import { transformers } from '../../../lib/transform/transformers' const _ = {} @@ -60,8 +60,8 @@ test('It should transform unprocessed asset with uploadFrom', () => { const transformedAsset = transformers.assets(assetMock, _) expect(transformedAsset.fields.file['en-US'].uploadFrom).toBeTruthy() expect(transformedAsset.fields.file['de-DE'].uploadFrom).toBeTruthy() - expect(transformedAsset.fields.file['en-US'].uploadFrom.sys.id).toBe(assetMock.fields.file['en-US'].uploadFrom.sys.id) - expect(transformedAsset.fields.file['de-DE'].uploadFrom.sys.id).toBe(assetMock.fields.file['de-DE'].uploadFrom.sys.id) + expect(transformedAsset.fields.file['en-US'].uploadFrom?.sys.id).toBe(assetMock.fields.file['en-US'].uploadFrom.sys.id) + expect(transformedAsset.fields.file['de-DE'].uploadFrom?.sys.id).toBe(assetMock.fields.file['de-DE'].uploadFrom.sys.id) }) test('It should transform webhook with credentials to normal webhook', () => {