From c38aa94e966b5a6e9117f5405703c5edbee72bd1 Mon Sep 17 00:00:00 2001 From: tasshi / Masaharu TASHIRO <33759872+mshrtsr@users.noreply.github.com> Date: Tue, 21 Feb 2023 16:38:49 +0900 Subject: [PATCH] feat: improve memory usage of record import (#218) --- src/record/import/index.ts | 35 ++-- src/record/import/parsers/index.ts | 19 -- .../__tests__/fixtures/emptyCsv/expected.ts | 3 - .../fixtures/withNoRecord/expected.ts | 3 - src/record/import/parsers/parseCsv/index.ts | 33 ---- src/record/import/parsers/parseCsv/record.ts | 85 -------- src/record/import/repositories/error.ts | 30 +++ .../localRecordRepositoryFromStream.ts | 31 +++ .../repositories/localRecordRepositoryMock.ts | 22 +++ .../parsers/__tests__/error.test.ts | 0 .../{ => repositories}/parsers/error.ts | 0 .../parseCsv/__tests__/fieldValue.test.ts | 4 +- .../__tests__/fixtures/emptyCsv/expected.ts | 3 + .../__tests__/fixtures/emptyCsv/index.ts | 0 .../__tests__/fixtures/emptyCsv/input.ts | 0 .../__tests__/fixtures/emptyCsv}/schema.ts | 2 +- .../__tests__/fixtures/withCr}/expected.ts | 12 +- .../__tests__/fixtures/withCr/index.ts | 0 .../__tests__/fixtures/withCr/input.ts | 0 .../__tests__/fixtures/withCr/schema.ts | 2 +- .../__tests__/fixtures/withCrLf/expected.ts | 12 +- .../__tests__/fixtures/withCrLf/index.ts | 0 .../__tests__/fixtures/withCrLf/input.ts | 0 .../__tests__/fixtures/withCrLf}/schema.ts | 2 +- .../__tests__/fixtures/withLf}/expected.ts | 12 +- .../__tests__/fixtures/withLf/index.ts | 0 .../__tests__/fixtures/withLf/input.ts | 0 .../__tests__/fixtures/withLf}/schema.ts | 2 +- .../fixtures/withMultipleSubtable/expected.ts | 20 +- .../fixtures/withMultipleSubtable/index.ts | 0 .../fixtures/withMultipleSubtable/input.ts | 0 .../fixtures/withMultipleSubtable/schema.ts | 2 +- .../fixtures/withNoRecord/expected.ts | 3 + .../__tests__/fixtures/withNoRecord/index.ts | 0 .../__tests__/fixtures/withNoRecord/input.ts | 0 .../fixtures/withNoRecord}/schema.ts | 2 +- .../fixtures/withSubtable/expected.ts | 12 +- .../__tests__/fixtures/withSubtable/index.ts | 0 .../__tests__/fixtures/withSubtable/input.ts | 0 .../__tests__/fixtures/withSubtable/schema.ts | 2 +- .../fixtures/withoutSubtable/expected.ts | 8 +- .../fixtures/withoutSubtable/index.ts | 0 .../fixtures/withoutSubtable/input.ts | 0 .../fixtures/withoutSubtable/schema.ts | 2 +- .../parsers/parseCsv/__tests__/index.test.ts | 22 ++- .../parsers/parseCsv/constants.ts | 0 .../parsers/parseCsv/field.ts | 6 +- .../parsers/parseCsv/fieldValue.ts | 4 +- .../repositories/parsers/parseCsv/index.ts | 60 ++++++ .../repositories/parsers/parseCsv/record.ts | 105 ++++++++++ .../parsers/parseCsv/subtable.ts | 6 +- src/record/import/types/record.ts | 2 +- .../usecases/__tests__/add/error.test.ts | 5 +- .../add/fixtures/can_upload_files.ts | 11 +- .../fixtures/can_upload_files_in_subtable.ts | 11 +- .../usecases/__tests__/add/index.test.ts | 25 ++- .../usecases/__tests__/upsert/error.test.ts | 5 +- .../upsertByNonExistentField/index.ts | 2 + .../upsertByNonExistentField/records.ts | 12 +- .../fixtures/upsertByNonUniqueKey/index.ts | 2 + .../fixtures/upsertByNonUniqueKey/records.ts | 12 +- .../upsert/fixtures/upsertByNumber/index.ts | 2 + .../upsert/fixtures/upsertByNumber/records.ts | 12 +- .../fixtures/upsertByRecordNumber/index.ts | 2 + .../fixtures/upsertByRecordNumber/records.ts | 12 +- .../upsertByRecordNumberWithAppCode/index.ts | 2 + .../records.ts | 12 +- .../index.ts | 2 + .../records.ts | 12 +- .../index.ts | 8 +- .../records.ts | 14 +- .../index.ts | 5 +- .../records.ts | 14 +- .../fixtures/upsertBySingleLineText/index.ts | 2 + .../upsertBySingleLineText/records.ts | 12 +- .../upsertByUnsupportedField/index.ts | 2 + .../upsertByUnsupportedField/records.ts | 12 +- .../upsertRecordsSequentially/index.ts | 2 + .../upsertRecordsSequentially/records.ts | 32 ++- .../upsertWithMissingFieldFromRecord/index.ts | 2 + .../records.ts | 12 +- .../index.ts | 2 + .../records.ts | 12 +- .../upsertWithMissingKeyFromRecord/index.ts | 2 + .../upsertWithMissingKeyFromRecord/records.ts | 12 +- .../usecases/__tests__/upsert/index.test.ts | 14 +- src/record/import/usecases/add.ts | 49 ++--- src/record/import/usecases/add/error.ts | 6 +- src/record/import/usecases/add/record.ts | 4 +- src/record/import/usecases/interface.ts | 8 + src/record/import/usecases/upsert.ts | 79 +++----- src/record/import/usecases/upsert/error.ts | 6 +- .../import/usecases/upsert/updateKey.ts | 20 +- .../import/utils/__tests__/error.test.ts | 5 +- src/record/import/utils/error.ts | 6 +- src/utils/__tests__/iterator.test.ts | 186 ++++++++++++++++++ src/utils/file.ts | 17 +- src/utils/iterator.ts | 87 ++++++++ src/utils/log.ts | 6 +- 99 files changed, 943 insertions(+), 404 deletions(-) delete mode 100644 src/record/import/parsers/index.ts delete mode 100644 src/record/import/parsers/parseCsv/__tests__/fixtures/emptyCsv/expected.ts delete mode 100644 src/record/import/parsers/parseCsv/__tests__/fixtures/withNoRecord/expected.ts delete mode 100644 src/record/import/parsers/parseCsv/index.ts delete mode 100644 src/record/import/parsers/parseCsv/record.ts create mode 100644 src/record/import/repositories/error.ts create mode 100644 src/record/import/repositories/localRecordRepositoryFromStream.ts create mode 100644 src/record/import/repositories/localRecordRepositoryMock.ts rename src/record/import/{ => repositories}/parsers/__tests__/error.test.ts (100%) rename src/record/import/{ => repositories}/parsers/error.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fieldValue.test.ts (96%) create mode 100644 src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/emptyCsv/expected.ts rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/emptyCsv/index.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/emptyCsv/input.ts (100%) rename src/record/import/{parsers/parseCsv/__tests__/fixtures/withNoRecord => repositories/parsers/parseCsv/__tests__/fixtures/emptyCsv}/schema.ts (98%) rename src/record/import/{parsers/parseCsv/__tests__/fixtures/withLf => repositories/parsers/parseCsv/__tests__/fixtures/withCr}/expected.ts (63%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withCr/index.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withCr/input.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withCr/schema.ts (94%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withCrLf/expected.ts (63%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withCrLf/index.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withCrLf/input.ts (100%) rename src/record/import/{parsers/parseCsv/__tests__/fixtures/withLf => repositories/parsers/parseCsv/__tests__/fixtures/withCrLf}/schema.ts (94%) rename src/record/import/{parsers/parseCsv/__tests__/fixtures/withCr => repositories/parsers/parseCsv/__tests__/fixtures/withLf}/expected.ts (63%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withLf/index.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withLf/input.ts (100%) rename src/record/import/{parsers/parseCsv/__tests__/fixtures/withCrLf => repositories/parsers/parseCsv/__tests__/fixtures/withLf}/schema.ts (94%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/expected.ts (80%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/index.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/input.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/schema.ts (96%) create mode 100644 src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withNoRecord/expected.ts rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withNoRecord/index.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withNoRecord/input.ts (100%) rename src/record/import/{parsers/parseCsv/__tests__/fixtures/emptyCsv => repositories/parsers/parseCsv/__tests__/fixtures/withNoRecord}/schema.ts (98%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withSubtable/expected.ts (95%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withSubtable/index.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withSubtable/input.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withSubtable/schema.ts (98%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withoutSubtable/expected.ts (89%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withoutSubtable/index.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withoutSubtable/input.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/fixtures/withoutSubtable/schema.ts (98%) rename src/record/import/{ => repositories}/parsers/parseCsv/__tests__/index.test.ts (61%) rename src/record/import/{ => repositories}/parsers/parseCsv/constants.ts (100%) rename src/record/import/{ => repositories}/parsers/parseCsv/field.ts (76%) rename src/record/import/{ => repositories}/parsers/parseCsv/fieldValue.ts (91%) create mode 100644 src/record/import/repositories/parsers/parseCsv/index.ts create mode 100644 src/record/import/repositories/parsers/parseCsv/record.ts rename src/record/import/{ => repositories}/parsers/parseCsv/subtable.ts (93%) create mode 100644 src/record/import/usecases/interface.ts create mode 100644 src/utils/__tests__/iterator.test.ts create mode 100644 src/utils/iterator.ts diff --git a/src/record/import/index.ts b/src/record/import/index.ts index 33b27c3036..cc7fc64eb2 100644 --- a/src/record/import/index.ts +++ b/src/record/import/index.ts @@ -1,14 +1,14 @@ import type { RestAPIClientOptions } from "../../kintone/client"; import { buildRestAPIClient } from "../../kintone/client"; import type { SupportedImportEncoding } from "../../utils/file"; -import { readFile } from "../../utils/file"; -import { parseRecords } from "./parsers"; +import { extractFileFormat, openFsStreamWithEncode } from "../../utils/file"; import { addRecords } from "./usecases/add"; import { upsertRecords } from "./usecases/upsert"; import { createSchema } from "./schema"; import { noop as defaultTransformer } from "./schema/transformers/noop"; import { userSelected } from "./schema/transformers/userSelected"; import { logger } from "../../utils/log"; +import { LocalRecordRepositoryFromStream } from "./repositories/localRecordRepositoryFromStream"; export type Options = { app: string; @@ -42,24 +42,33 @@ export const run: ( ? userSelected(fields, fieldsJson, updateKey) : defaultTransformer() ); - const { content, format } = await readFile(filePath, encoding); - const records = await parseRecords({ - source: content, + const format = extractFileFormat(filePath); + const localRecordRepository = new LocalRecordRepositoryFromStream( + () => openFsStreamWithEncode(filePath, encoding), format, - schema, - }); - if (records.length === 0) { + schema + ); + + if ((await localRecordRepository.length()) === 0) { logger.warn("The input file does not have any records"); return; } + const skipMissingFields = !fields; if (updateKey) { - await upsertRecords(apiClient, app, records, schema, updateKey, { - attachmentsDir, - skipMissingFields, - }); + await upsertRecords( + apiClient, + app, + localRecordRepository, + schema, + updateKey, + { + attachmentsDir, + skipMissingFields, + } + ); } else { - await addRecords(apiClient, app, records, schema, { + await addRecords(apiClient, app, localRecordRepository, schema, { attachmentsDir, skipMissingFields, }); diff --git a/src/record/import/parsers/index.ts b/src/record/import/parsers/index.ts deleted file mode 100644 index 757e0ecba3..0000000000 --- a/src/record/import/parsers/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { KintoneRecord } from "../types/record"; -import type { RecordSchema } from "../types/schema"; - -import { parseCsv } from "./parseCsv"; -import { ParserError } from "./error"; - -export const parseRecords: (options: { - source: string; - format: string; - schema: RecordSchema; -}) => Promise = async (options) => { - const { source, format, schema } = options; - switch (format) { - case "csv": - return parseCsv(source, schema); - default: - throw new ParserError(`Unexpected file type: ${format} is unacceptable.`); - } -}; diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/emptyCsv/expected.ts b/src/record/import/parsers/parseCsv/__tests__/fixtures/emptyCsv/expected.ts deleted file mode 100644 index 7baed26095..0000000000 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/emptyCsv/expected.ts +++ /dev/null @@ -1,3 +0,0 @@ -import type { KintoneRecord } from "../../../../../types/record"; - -export const expected: KintoneRecord[] = []; diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withNoRecord/expected.ts b/src/record/import/parsers/parseCsv/__tests__/fixtures/withNoRecord/expected.ts deleted file mode 100644 index 7baed26095..0000000000 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withNoRecord/expected.ts +++ /dev/null @@ -1,3 +0,0 @@ -import type { KintoneRecord } from "../../../../../types/record"; - -export const expected: KintoneRecord[] = []; diff --git a/src/record/import/parsers/parseCsv/index.ts b/src/record/import/parsers/parseCsv/index.ts deleted file mode 100644 index 077540cab4..0000000000 --- a/src/record/import/parsers/parseCsv/index.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { CsvRow } from "../../../../kintone/types"; -import type { KintoneRecord } from "../../types/record"; -import type { RecordSchema } from "../../types/schema"; - -import csvParse from "csv-parse/lib/sync"; - -import { convertRecord, recordReader } from "./record"; -import { SEPARATOR } from "./constants"; -import { ParserError } from "../error"; - -export const parseCsv: ( - csv: string, - schema: RecordSchema -) => KintoneRecord[] = (csv, schema) => { - try { - const rows: CsvRow[] = csvParse(csv, { - columns: true, - skip_empty_lines: true, - delimiter: SEPARATOR, - }); - - const records: KintoneRecord[] = []; - - for (const recordRows of recordReader(rows)) { - const record = convertRecord(recordRows, schema); - records.push(record); - } - - return records; - } catch (e) { - throw new ParserError(e); - } -}; diff --git a/src/record/import/parsers/parseCsv/record.ts b/src/record/import/parsers/parseCsv/record.ts deleted file mode 100644 index df9248a60c..0000000000 --- a/src/record/import/parsers/parseCsv/record.ts +++ /dev/null @@ -1,85 +0,0 @@ -import type { CsvRow } from "../../../../kintone/types"; -import type { KintoneRecord } from "../../types/record"; -import type { RecordSchema } from "../../types/schema"; - -import { convertField, fieldReader } from "./field"; -import { convertSubtableField, subtableFieldReader } from "./subtable"; -import { PRIMARY_MARK } from "./constants"; - -type RecordCsv = { - rows: CsvRow[]; - firstRowIndex: number; - lastRowIndex: number; -}; - -export const convertRecord = ( - recordCsv: RecordCsv, - schema: RecordSchema -): KintoneRecord => { - const recordData: KintoneRecord["data"] = {}; - for (const field of fieldReader(recordCsv.rows[0], schema)) { - recordData[field.code] = convertField(field); - } - for (const subtableField of subtableFieldReader(recordCsv.rows, schema)) { - recordData[subtableField.code] = convertSubtableField(subtableField); - } - return { - data: recordData, - metadata: { - format: { - type: "csv", - firstRowIndex: recordCsv.firstRowIndex, - lastRowIndex: recordCsv.lastRowIndex, - }, - }, - }; -}; - -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/Arrow_functions#use_of_the_yield_keyword -// eslint-disable-next-line func-style -export function* recordReader( - rows: CsvRow[] -): Generator { - if (rows.length === 0) { - return; - } - - const lineOffset = 1; // offset the header row - - if (!hasSubtable(rows[0])) { - yield* rows.map((row, index) => ({ - rows: [row], - firstRowIndex: index + lineOffset, - lastRowIndex: index + lineOffset, - })); - return; - } - - let index = 0; - while (index < rows.length) { - let first = index; - let last = first; - - // skip to the first primary mark - while (first < rows.length && !isPrimaryCsvRow(rows[first])) { - first++; - } - - // find the row just before the next primary mark - while (last + 1 < rows.length && !isPrimaryCsvRow(rows[last + 1])) { - last++; - } - - yield { - rows: rows.slice(first, last + 1), - firstRowIndex: first + lineOffset, - lastRowIndex: last + lineOffset, - }; - - index = last + 1; - } -} - -const hasSubtable = (row: CsvRow): boolean => PRIMARY_MARK in row; - -const isPrimaryCsvRow = (row: CsvRow): boolean => !!row[PRIMARY_MARK]; diff --git a/src/record/import/repositories/error.ts b/src/record/import/repositories/error.ts new file mode 100644 index 0000000000..4cfff62a4b --- /dev/null +++ b/src/record/import/repositories/error.ts @@ -0,0 +1,30 @@ +import { ParserError } from "./parsers/error"; + +export class RepositoryError extends Error { + private readonly cause: unknown; + + constructor(cause: unknown) { + const message = + "An error occurred while loading records from the data source"; + super(message); + + this.name = "RepositoryError"; + this.message = message; + this.cause = cause; + + // https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work + // Set the prototype explicitly. + Object.setPrototypeOf(this, RepositoryError.prototype); + } + + toString(): string { + let errorMessage = ""; + errorMessage += this.message + "\n"; + if (this.cause instanceof ParserError) { + errorMessage += this.cause.toString(); + } else { + errorMessage += this.cause + "\n"; + } + return errorMessage; + } +} diff --git a/src/record/import/repositories/localRecordRepositoryFromStream.ts b/src/record/import/repositories/localRecordRepositoryFromStream.ts new file mode 100644 index 0000000000..444fbdf2d3 --- /dev/null +++ b/src/record/import/repositories/localRecordRepositoryFromStream.ts @@ -0,0 +1,31 @@ +import type { LocalRecordRepository } from "../usecases/interface"; +import type { RecordSchema } from "../types/schema"; +import { RepositoryError } from "./error"; +import { countRecordsFromCsv, csvReader } from "./parsers/parseCsv"; +import type { LocalRecord } from "../types/record"; + +export class LocalRecordRepositoryFromStream implements LocalRecordRepository { + readonly format: string; + readonly length: () => Promise; + + readonly reader: () => AsyncGenerator; + + constructor( + source: () => NodeJS.ReadableStream, + format: string, + schema: RecordSchema + ) { + this.format = format; + this.length = () => countRecordsFromCsv(source()); + + switch (format) { + case "csv": + this.reader = () => csvReader(source, schema); + break; + default: + throw new RepositoryError( + `Unexpected file type: ${format} is unacceptable.` + ); + } + } +} diff --git a/src/record/import/repositories/localRecordRepositoryMock.ts b/src/record/import/repositories/localRecordRepositoryMock.ts new file mode 100644 index 0000000000..1eb4b9c125 --- /dev/null +++ b/src/record/import/repositories/localRecordRepositoryMock.ts @@ -0,0 +1,22 @@ +import type { LocalRecordRepository } from "../usecases/interface"; +import type { LocalRecord } from "../types/record"; + +export class LocalRecordRepositoryMock implements LocalRecordRepository { + readonly format: string; + readonly length: () => Promise; + + readonly reader: () => AsyncGenerator; + + constructor(source: LocalRecord[], format: string) { + this.format = format; + this.length = async () => source.length; + this.reader = () => asyncGeneratorFromStream(source); + } +} + +// eslint-disable-next-line func-style +async function* asyncGeneratorFromStream( + source: LocalRecord[] +): AsyncGenerator { + yield* source; +} diff --git a/src/record/import/parsers/__tests__/error.test.ts b/src/record/import/repositories/parsers/__tests__/error.test.ts similarity index 100% rename from src/record/import/parsers/__tests__/error.test.ts rename to src/record/import/repositories/parsers/__tests__/error.test.ts diff --git a/src/record/import/parsers/error.ts b/src/record/import/repositories/parsers/error.ts similarity index 100% rename from src/record/import/parsers/error.ts rename to src/record/import/repositories/parsers/error.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fieldValue.test.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fieldValue.test.ts similarity index 96% rename from src/record/import/parsers/parseCsv/__tests__/fieldValue.test.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fieldValue.test.ts index d9c107a7e9..7631e389bc 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fieldValue.test.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fieldValue.test.ts @@ -1,5 +1,5 @@ -import type * as Fields from "../../../types/field"; -import type { FieldSchema } from "../../../types/schema"; +import type * as Fields from "../../../../types/field"; +import type { FieldSchema } from "../../../../types/schema"; import { convertFieldValue } from "../fieldValue"; diff --git a/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/emptyCsv/expected.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/emptyCsv/expected.ts new file mode 100644 index 0000000000..fcbb883210 --- /dev/null +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/emptyCsv/expected.ts @@ -0,0 +1,3 @@ +import type { LocalRecord } from "../../../../../../types/record"; + +export const expected: LocalRecord[] = []; diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/emptyCsv/index.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/emptyCsv/index.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/emptyCsv/index.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/emptyCsv/index.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/emptyCsv/input.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/emptyCsv/input.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/emptyCsv/input.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/emptyCsv/input.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withNoRecord/schema.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/emptyCsv/schema.ts similarity index 98% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withNoRecord/schema.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/emptyCsv/schema.ts index 0cb5b1f232..19778334a1 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withNoRecord/schema.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/emptyCsv/schema.ts @@ -1,4 +1,4 @@ -import type { RecordSchema } from "../../../../../types/schema"; +import type { RecordSchema } from "../../../../../../types/schema"; export const schema: RecordSchema = { fields: [ diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withLf/expected.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCr/expected.ts similarity index 63% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withLf/expected.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCr/expected.ts index 24aa92fc22..9a2b9ae118 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withLf/expected.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCr/expected.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../../types/record"; -export const expected: KintoneRecord[] = [ +export const expected: LocalRecord[] = [ { data: { singleLineText: { @@ -13,7 +13,9 @@ export const expected: KintoneRecord[] = [ value: ['"sample3"', "sample4,sample5"], }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -27,6 +29,8 @@ export const expected: KintoneRecord[] = [ value: ['"sample4"', "sample5,sample6"], }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withCr/index.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCr/index.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withCr/index.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCr/index.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withCr/input.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCr/input.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withCr/input.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCr/input.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withCr/schema.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCr/schema.ts similarity index 94% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withCr/schema.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCr/schema.ts index 77c8703cc6..bc4fddc348 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withCr/schema.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCr/schema.ts @@ -1,4 +1,4 @@ -import type { RecordSchema } from "../../../../../types/schema"; +import type { RecordSchema } from "../../../../../../types/schema"; export const schema: RecordSchema = { fields: [ diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withCrLf/expected.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCrLf/expected.ts similarity index 63% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withCrLf/expected.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCrLf/expected.ts index 24aa92fc22..9a2b9ae118 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withCrLf/expected.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCrLf/expected.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../../types/record"; -export const expected: KintoneRecord[] = [ +export const expected: LocalRecord[] = [ { data: { singleLineText: { @@ -13,7 +13,9 @@ export const expected: KintoneRecord[] = [ value: ['"sample3"', "sample4,sample5"], }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -27,6 +29,8 @@ export const expected: KintoneRecord[] = [ value: ['"sample4"', "sample5,sample6"], }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withCrLf/index.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCrLf/index.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withCrLf/index.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCrLf/index.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withCrLf/input.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCrLf/input.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withCrLf/input.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCrLf/input.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withLf/schema.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCrLf/schema.ts similarity index 94% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withLf/schema.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCrLf/schema.ts index 77c8703cc6..bc4fddc348 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withLf/schema.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withCrLf/schema.ts @@ -1,4 +1,4 @@ -import type { RecordSchema } from "../../../../../types/schema"; +import type { RecordSchema } from "../../../../../../types/schema"; export const schema: RecordSchema = { fields: [ diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withCr/expected.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withLf/expected.ts similarity index 63% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withCr/expected.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withLf/expected.ts index 24aa92fc22..9a2b9ae118 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withCr/expected.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withLf/expected.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../../types/record"; -export const expected: KintoneRecord[] = [ +export const expected: LocalRecord[] = [ { data: { singleLineText: { @@ -13,7 +13,9 @@ export const expected: KintoneRecord[] = [ value: ['"sample3"', "sample4,sample5"], }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -27,6 +29,8 @@ export const expected: KintoneRecord[] = [ value: ['"sample4"', "sample5,sample6"], }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withLf/index.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withLf/index.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withLf/index.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withLf/index.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withLf/input.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withLf/input.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withLf/input.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withLf/input.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withCrLf/schema.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withLf/schema.ts similarity index 94% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withCrLf/schema.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withLf/schema.ts index 77c8703cc6..bc4fddc348 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withCrLf/schema.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withLf/schema.ts @@ -1,4 +1,4 @@ -import type { RecordSchema } from "../../../../../types/schema"; +import type { RecordSchema } from "../../../../../../types/schema"; export const schema: RecordSchema = { fields: [ diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/expected.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/expected.ts similarity index 80% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/expected.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/expected.ts index 020917937a..5c927ac14d 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/expected.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/expected.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../../types/record"; -export const expected: KintoneRecord[] = [ +export const expected: LocalRecord[] = [ { data: { text: { @@ -59,7 +59,9 @@ export const expected: KintoneRecord[] = [ ], }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 4 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 4 }, + }, }, { data: { @@ -71,7 +73,9 @@ export const expected: KintoneRecord[] = [ value: [{ id: "202", value: {} }], }, }, - metadata: { format: { type: "csv", firstRowIndex: 5, lastRowIndex: 6 } }, + metadata: { + format: { type: "csv", firstRowIndex: 5, lastRowIndex: 6 }, + }, }, { data: { @@ -101,12 +105,16 @@ export const expected: KintoneRecord[] = [ ], }, }, - metadata: { format: { type: "csv", firstRowIndex: 7, lastRowIndex: 8 } }, + metadata: { + format: { type: "csv", firstRowIndex: 7, lastRowIndex: 8 }, + }, }, { data: { text: { value: "sample1" }, }, - metadata: { format: { type: "csv", firstRowIndex: 9, lastRowIndex: 9 } }, + metadata: { + format: { type: "csv", firstRowIndex: 9, lastRowIndex: 9 }, + }, }, ]; diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/index.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/index.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/index.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/index.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/input.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/input.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/input.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/input.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/schema.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/schema.ts similarity index 96% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/schema.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/schema.ts index 509c7a3223..c4d5786cd2 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/schema.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withMultipleSubtable/schema.ts @@ -1,4 +1,4 @@ -import type { RecordSchema } from "../../../../../types/schema"; +import type { RecordSchema } from "../../../../../../types/schema"; export const schema: RecordSchema = { fields: [ diff --git a/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withNoRecord/expected.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withNoRecord/expected.ts new file mode 100644 index 0000000000..fcbb883210 --- /dev/null +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withNoRecord/expected.ts @@ -0,0 +1,3 @@ +import type { LocalRecord } from "../../../../../../types/record"; + +export const expected: LocalRecord[] = []; diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withNoRecord/index.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withNoRecord/index.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withNoRecord/index.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withNoRecord/index.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withNoRecord/input.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withNoRecord/input.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withNoRecord/input.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withNoRecord/input.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/emptyCsv/schema.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withNoRecord/schema.ts similarity index 98% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/emptyCsv/schema.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withNoRecord/schema.ts index 0cb5b1f232..19778334a1 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/emptyCsv/schema.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withNoRecord/schema.ts @@ -1,4 +1,4 @@ -import type { RecordSchema } from "../../../../../types/schema"; +import type { RecordSchema } from "../../../../../../types/schema"; export const schema: RecordSchema = { fields: [ diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withSubtable/expected.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withSubtable/expected.ts similarity index 95% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withSubtable/expected.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withSubtable/expected.ts index a7723ccdc8..341bad9202 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withSubtable/expected.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withSubtable/expected.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../../types/record"; -export const expected: KintoneRecord[] = [ +export const expected: LocalRecord[] = [ { data: { updatedTime: { @@ -137,7 +137,9 @@ export const expected: KintoneRecord[] = [ ], }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 2 }, + }, }, { data: { @@ -275,6 +277,8 @@ export const expected: KintoneRecord[] = [ ], }, }, - metadata: { format: { type: "csv", firstRowIndex: 3, lastRowIndex: 4 } }, + metadata: { + format: { type: "csv", firstRowIndex: 3, lastRowIndex: 4 }, + }, }, ]; diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withSubtable/index.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withSubtable/index.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withSubtable/index.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withSubtable/index.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withSubtable/input.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withSubtable/input.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withSubtable/input.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withSubtable/input.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withSubtable/schema.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withSubtable/schema.ts similarity index 98% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withSubtable/schema.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withSubtable/schema.ts index 0cb5b1f232..19778334a1 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withSubtable/schema.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withSubtable/schema.ts @@ -1,4 +1,4 @@ -import type { RecordSchema } from "../../../../../types/schema"; +import type { RecordSchema } from "../../../../../../types/schema"; export const schema: RecordSchema = { fields: [ diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withoutSubtable/expected.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withoutSubtable/expected.ts similarity index 89% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withoutSubtable/expected.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withoutSubtable/expected.ts index 2942f7ca35..992317e444 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withoutSubtable/expected.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withoutSubtable/expected.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../../types/record"; -export const expected: KintoneRecord[] = [ +export const expected: LocalRecord[] = [ { data: { dropDown: { @@ -84,6 +84,8 @@ export const expected: KintoneRecord[] = [ value: "16:39", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, ]; diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withoutSubtable/index.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withoutSubtable/index.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withoutSubtable/index.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withoutSubtable/index.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withoutSubtable/input.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withoutSubtable/input.ts similarity index 100% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withoutSubtable/input.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withoutSubtable/input.ts diff --git a/src/record/import/parsers/parseCsv/__tests__/fixtures/withoutSubtable/schema.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withoutSubtable/schema.ts similarity index 98% rename from src/record/import/parsers/parseCsv/__tests__/fixtures/withoutSubtable/schema.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withoutSubtable/schema.ts index 630693928d..1bebff7b70 100644 --- a/src/record/import/parsers/parseCsv/__tests__/fixtures/withoutSubtable/schema.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/fixtures/withoutSubtable/schema.ts @@ -1,4 +1,4 @@ -import type { RecordSchema } from "../../../../../types/schema"; +import type { RecordSchema } from "../../../../../../types/schema"; export const schema: RecordSchema = { fields: [ diff --git a/src/record/import/parsers/parseCsv/__tests__/index.test.ts b/src/record/import/repositories/parsers/parseCsv/__tests__/index.test.ts similarity index 61% rename from src/record/import/parsers/parseCsv/__tests__/index.test.ts rename to src/record/import/repositories/parsers/parseCsv/__tests__/index.test.ts index c443c8112e..aedb3ed314 100644 --- a/src/record/import/parsers/parseCsv/__tests__/index.test.ts +++ b/src/record/import/repositories/parsers/parseCsv/__tests__/index.test.ts @@ -1,7 +1,7 @@ -import type { KintoneRecord } from "../../../types/record"; -import type { RecordSchema } from "../../../types/schema"; +import type { LocalRecord } from "../../../../types/record"; +import type { RecordSchema } from "../../../../types/schema"; -import { parseCsv } from "../index"; +import { csvReader } from "../index"; import { pattern as withoutSubtable } from "./fixtures/withoutSubtable"; import { pattern as withSubtable } from "./fixtures/withSubtable"; @@ -11,12 +11,13 @@ import { pattern as withNoRecord } from "./fixtures/withNoRecord"; import { pattern as withCrLf } from "./fixtures/withCrLf"; import { pattern as withCr } from "./fixtures/withCr"; import { pattern as withLf } from "./fixtures/withLf"; +import { Readable } from "stream"; export type TestPattern = { description: string; schema: RecordSchema; input: string; - expected: KintoneRecord[]; + expected: LocalRecord[]; }; describe("parseCsv", () => { @@ -30,9 +31,14 @@ describe("parseCsv", () => { withCr, withLf, ]; - it.each(patterns)("$description", (pattern) => { - expect(parseCsv(pattern.input, pattern.schema)).toStrictEqual( - pattern.expected - ); + it.each(patterns)("$description", async (pattern) => { + const records = []; + for await (const record of csvReader( + () => Readable.from(pattern.input), + pattern.schema + )) { + records.push(record); + } + expect(records).toStrictEqual(pattern.expected); }); }); diff --git a/src/record/import/parsers/parseCsv/constants.ts b/src/record/import/repositories/parsers/parseCsv/constants.ts similarity index 100% rename from src/record/import/parsers/parseCsv/constants.ts rename to src/record/import/repositories/parsers/parseCsv/constants.ts diff --git a/src/record/import/parsers/parseCsv/field.ts b/src/record/import/repositories/parsers/parseCsv/field.ts similarity index 76% rename from src/record/import/parsers/parseCsv/field.ts rename to src/record/import/repositories/parsers/parseCsv/field.ts index fde89f2fb0..296af050d2 100644 --- a/src/record/import/parsers/parseCsv/field.ts +++ b/src/record/import/repositories/parsers/parseCsv/field.ts @@ -1,6 +1,6 @@ -import type { CsvRow } from "../../../../kintone/types"; -import type * as Fields from "../../types/field"; -import type { FieldSchema, RecordSchema } from "../../types/schema"; +import type { CsvRow } from "../../../../../kintone/types"; +import type * as Fields from "../../../types/field"; +import type { FieldSchema, RecordSchema } from "../../../types/schema"; import { convertFieldValue } from "./fieldValue"; diff --git a/src/record/import/parsers/parseCsv/fieldValue.ts b/src/record/import/repositories/parsers/parseCsv/fieldValue.ts similarity index 91% rename from src/record/import/parsers/parseCsv/fieldValue.ts rename to src/record/import/repositories/parsers/parseCsv/fieldValue.ts index f967e9bbd1..68d57c5a1b 100644 --- a/src/record/import/parsers/parseCsv/fieldValue.ts +++ b/src/record/import/repositories/parsers/parseCsv/fieldValue.ts @@ -1,5 +1,5 @@ -import type * as Fields from "../../types/field"; -import type { FieldSchema } from "../../types/schema"; +import type * as Fields from "../../../types/field"; +import type { FieldSchema } from "../../../types/schema"; import { LINE_BREAK } from "./constants"; diff --git a/src/record/import/repositories/parsers/parseCsv/index.ts b/src/record/import/repositories/parsers/parseCsv/index.ts new file mode 100644 index 0000000000..de84904506 --- /dev/null +++ b/src/record/import/repositories/parsers/parseCsv/index.ts @@ -0,0 +1,60 @@ +import type { RecordSchema } from "../../../types/schema"; + +import csvParse from "csv-parse"; + +import { convertRecord, recordReader } from "./record"; +import { SEPARATOR } from "./constants"; +import { ParserError } from "../error"; +import type { LocalRecordRepository } from "../../../usecases/interface"; + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/Arrow_functions#use_of_the_yield_keyword +// eslint-disable-next-line func-style +export async function* csvReader( + source: () => NodeJS.ReadableStream, + schema: RecordSchema +): ReturnType { + try { + const sourceStream = source(); + const csvStream = source().pipe( + csvParse({ + columns: true, + skip_empty_lines: true, + delimiter: SEPARATOR, + }) + ); + sourceStream.on("error", (e) => { + csvStream.destroy(e); + }); + + for await (const recordRows of recordReader(csvStream)) { + yield convertRecord(recordRows, schema); + } + } catch (e) { + throw new ParserError(e); + } +} + +export const countRecordsFromCsv = async ( + source: NodeJS.ReadableStream +): Promise => { + try { + const csvStream = source.pipe( + csvParse({ + columns: true, + skip_empty_lines: true, + delimiter: SEPARATOR, + }) + ); + source.on("error", (e) => { + csvStream.destroy(e); + }); + + let count = 0; + for await (const recordRows of recordReader(csvStream)) { + count++; + } + return count; + } catch (e) { + throw new ParserError(e); + } +}; diff --git a/src/record/import/repositories/parsers/parseCsv/record.ts b/src/record/import/repositories/parsers/parseCsv/record.ts new file mode 100644 index 0000000000..23ecbae8c7 --- /dev/null +++ b/src/record/import/repositories/parsers/parseCsv/record.ts @@ -0,0 +1,105 @@ +import type { CsvRow } from "../../../../../kintone/types"; +import type { LocalRecord } from "../../../types/record"; +import type { RecordSchema } from "../../../types/schema"; + +import { convertField, fieldReader } from "./field"; +import { convertSubtableField, subtableFieldReader } from "./subtable"; +import { PRIMARY_MARK } from "./constants"; +import type csvParse from "csv-parse"; +import { Readable } from "stream"; +import { withIndex, withNext } from "../../../../../utils/iterator"; + +type RecordCsv = { + rows: CsvRow[]; + firstRowIndex: number; + lastRowIndex: number; +}; + +export const convertRecord = ( + recordCsv: RecordCsv, + schema: RecordSchema +): LocalRecord => { + const recordData: LocalRecord["data"] = {}; + for (const field of fieldReader(recordCsv.rows[0], schema)) { + recordData[field.code] = convertField(field); + } + for (const subtableField of subtableFieldReader(recordCsv.rows, schema)) { + recordData[subtableField.code] = convertSubtableField(subtableField); + } + return { + data: recordData, + metadata: { + format: { + type: "csv", + firstRowIndex: recordCsv.firstRowIndex, + lastRowIndex: recordCsv.lastRowIndex, + }, + }, + }; +}; + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/Arrow_functions#use_of_the_yield_keyword +// eslint-disable-next-line func-style +export async function* recordReader( + csvStream: csvParse.Parser +): AsyncGenerator { + const lineOffset = 1; // offset the header row + + const { value: firstRow }: { value: CsvRow | null } = await csvStream[ + Symbol.asyncIterator + ]().next(); + + if (firstRow === null || firstRow === undefined) { + return; + } + + const stream = unshiftToStream(csvStream, firstRow); + const generator = withIndex(withNext(stream[Symbol.asyncIterator]())); + + if (!hasSubtable(firstRow)) { + for await (const { + data: { current: row }, + index: rowIndex, + } of generator) { + yield { + rows: [row], + firstRowIndex: rowIndex + lineOffset, + lastRowIndex: rowIndex + lineOffset, + }; + } + return; + } + + let rows: CsvRow[] = []; + let firstRowIndex = 0; + + for await (const { + data: { current: currentRow, next: nextRow }, + index: rowIndex, + } of generator) { + rows.push(currentRow); + if (nextRow === undefined || isPrimaryCsvRow(nextRow)) { + if (rows.length > 0 && isPrimaryCsvRow(rows[0])) { + yield { + rows: rows, + firstRowIndex: firstRowIndex + lineOffset, + lastRowIndex: rowIndex + lineOffset, + }; + } + firstRowIndex = rowIndex + 1; + rows = []; + } + } +} + +const hasSubtable = (row: CsvRow): boolean => PRIMARY_MARK in row; + +const isPrimaryCsvRow = (row: CsvRow): boolean => !!row[PRIMARY_MARK]; + +const unshiftToStream = (stream: Readable, element: unknown) => + Readable.from( + (async function* () { + yield element; + yield* stream; + })() + ); diff --git a/src/record/import/parsers/parseCsv/subtable.ts b/src/record/import/repositories/parsers/parseCsv/subtable.ts similarity index 93% rename from src/record/import/parsers/parseCsv/subtable.ts rename to src/record/import/repositories/parsers/parseCsv/subtable.ts index ad4f357e40..51b05aff39 100644 --- a/src/record/import/parsers/parseCsv/subtable.ts +++ b/src/record/import/repositories/parsers/parseCsv/subtable.ts @@ -1,6 +1,6 @@ -import type { CsvRow } from "../../../../kintone/types"; -import type * as Fields from "../../types/field"; -import type { InSubtable, RecordSchema } from "../../types/schema"; +import type { CsvRow } from "../../../../../kintone/types"; +import type * as Fields from "../../../types/field"; +import type { InSubtable, RecordSchema } from "../../../types/schema"; import { convertFieldValue } from "./fieldValue"; diff --git a/src/record/import/types/record.ts b/src/record/import/types/record.ts index 1969156e9c..0d76f4e7ce 100644 --- a/src/record/import/types/record.ts +++ b/src/record/import/types/record.ts @@ -1,6 +1,6 @@ import type { OneOf } from "./field"; -export type KintoneRecord = { +export type LocalRecord = { data: { [fieldCode: string]: OneOf }; metadata: { format: { diff --git a/src/record/import/usecases/__tests__/add/error.test.ts b/src/record/import/usecases/__tests__/add/error.test.ts index 2ac91b32f2..902d9ab925 100644 --- a/src/record/import/usecases/__tests__/add/error.test.ts +++ b/src/record/import/usecases/__tests__/add/error.test.ts @@ -1,4 +1,4 @@ -import type { KintoneRecord } from "../../../types/record"; +import type { LocalRecord } from "../../../types/record"; import type { KintoneErrorResponse } from "@kintone/rest-api-client"; import { KintoneAllRecordsError, @@ -34,10 +34,11 @@ describe("AddRecordsError", () => { const numOfAlreadyImportedRecords = 10; const numOfProcessedRecords = 30; const errorIndex = 44; - const records: KintoneRecord[] = [...Array(numOfAllRecords).keys()].map( + const records: LocalRecord[] = [...Array(numOfAllRecords).keys()].map( (index) => ({ data: {}, metadata: { + recordIndex: index, format: { type: "csv", firstRowIndex: index + 1, diff --git a/src/record/import/usecases/__tests__/add/fixtures/can_upload_files.ts b/src/record/import/usecases/__tests__/add/fixtures/can_upload_files.ts index def3c93996..59a088345a 100644 --- a/src/record/import/usecases/__tests__/add/fixtures/can_upload_files.ts +++ b/src/record/import/usecases/__tests__/add/fixtures/can_upload_files.ts @@ -1,10 +1,11 @@ -import type { KintoneRecord } from "../../../../types/record"; +import type { LocalRecord } from "../../../../types/record"; import type { KintoneRecordForParameter } from "../../../../../../kintone/types"; import type { RecordSchema } from "../../../../types/schema"; import path from "path"; +import { LocalRecordRepositoryMock } from "../../../../repositories/localRecordRepositoryMock"; -export const input: KintoneRecord[] = [ +export const inputRecords: LocalRecord[] = [ { data: { singleLineText: { @@ -21,10 +22,14 @@ export const input: KintoneRecord[] = [ ], }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, ]; +export const input = new LocalRecordRepositoryMock(inputRecords, "csv"); + export const expected: KintoneRecordForParameter[] = [ { singleLineText: { diff --git a/src/record/import/usecases/__tests__/add/fixtures/can_upload_files_in_subtable.ts b/src/record/import/usecases/__tests__/add/fixtures/can_upload_files_in_subtable.ts index 1e64e7e36b..838b8332ae 100644 --- a/src/record/import/usecases/__tests__/add/fixtures/can_upload_files_in_subtable.ts +++ b/src/record/import/usecases/__tests__/add/fixtures/can_upload_files_in_subtable.ts @@ -1,10 +1,11 @@ -import type { KintoneRecord } from "../../../../types/record"; +import type { LocalRecord } from "../../../../types/record"; import type { KintoneRecordForParameter } from "../../../../../../kintone/types"; import type { RecordSchema } from "../../../../types/schema"; import path from "path"; +import { LocalRecordRepositoryMock } from "../../../../repositories/localRecordRepositoryMock"; -export const input: KintoneRecord[] = [ +export const inputRecords: LocalRecord[] = [ { data: { singleLineText: { @@ -29,10 +30,14 @@ export const input: KintoneRecord[] = [ ], }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, ]; +export const input = new LocalRecordRepositoryMock(inputRecords, "csv"); + export const expected: KintoneRecordForParameter[] = [ { singleLineText: { diff --git a/src/record/import/usecases/__tests__/add/index.test.ts b/src/record/import/usecases/__tests__/add/index.test.ts index 2cefd85e65..680a75d6e2 100644 --- a/src/record/import/usecases/__tests__/add/index.test.ts +++ b/src/record/import/usecases/__tests__/add/index.test.ts @@ -1,5 +1,5 @@ import type { RecordSchema } from "../../../types/schema"; -import type { KintoneRecord } from "../../../types/record"; +import type { LocalRecord } from "../../../types/record"; import { KintoneRestAPIClient } from "@kintone/rest-api-client"; import { addRecords } from "../../add"; @@ -9,6 +9,8 @@ import path from "path"; import * as canUploadFiles from "./fixtures/can_upload_files"; import * as canUploadFilesInSubtable from "./fixtures/can_upload_files_in_subtable"; import { AddRecordsError } from "../../add/error"; +import { inputRecords } from "./fixtures/can_upload_files"; +import { LocalRecordRepositoryMock } from "../../../repositories/localRecordRepositoryMock"; describe("addRecords", () => { let apiClient: KintoneRestAPIClient; @@ -22,7 +24,13 @@ describe("addRecords", () => { it("should not fail", () => { apiClient.record.addAllRecords = jest.fn().mockResolvedValue([{}]); return expect( - addRecords(apiClient, "1", [], { fields: [] }, { attachmentsDir: "" }) + addRecords( + apiClient, + "1", + new LocalRecordRepositoryMock([], "csv"), + { fields: [] }, + { attachmentsDir: "" } + ) ).resolves.not.toThrow(); }); @@ -31,7 +39,7 @@ describe("addRecords", () => { apiClient.record.addAllRecords = addAllRecordsMockFn; const ATTACHMENTS_DIR = ""; const APP_ID = "1"; - const RECORDS: KintoneRecord[] = [ + const RECORDS: LocalRecord[] = [ { data: { number: { value: "1" } }, metadata: { @@ -58,8 +66,9 @@ describe("addRecords", () => { }, ], }; + const repository = new LocalRecordRepositoryMock(RECORDS, "csv"); - await addRecords(apiClient, APP_ID, RECORDS, SCHEMA, { + await addRecords(apiClient, APP_ID, repository, SCHEMA, { attachmentsDir: ATTACHMENTS_DIR, }); @@ -83,7 +92,7 @@ describe("addRecords", () => { ).rejects.toThrow( new AddRecordsError( new Error("--attachments-dir option is required."), - canUploadFiles.input, + inputRecords, 0, canUploadFiles.schema ) @@ -113,7 +122,8 @@ describe("addRecords", () => { ); // apiClient.file.uploadFile should be called with correct filePath - const fileInfos = canUploadFiles.input[0].data.attachment.value as Array<{ + const fileInfos = canUploadFiles.inputRecords[0].data.attachment + .value as Array<{ localFilePath: string; }>; expect(uploadFileMockFn.mock.calls[0][0]).toStrictEqual({ @@ -157,7 +167,8 @@ describe("addRecords", () => { ); // apiClient.file.uploadFile should be called with correct filePath - const fileInfos = canUploadFiles.input[0].data.attachment.value as Array<{ + const fileInfos = canUploadFiles.inputRecords[0].data.attachment + .value as Array<{ localFilePath: string; }>; expect(uploadFileMockFn.mock.calls[0][0]).toStrictEqual({ diff --git a/src/record/import/usecases/__tests__/upsert/error.test.ts b/src/record/import/usecases/__tests__/upsert/error.test.ts index 9419768001..1170ea56fc 100644 --- a/src/record/import/usecases/__tests__/upsert/error.test.ts +++ b/src/record/import/usecases/__tests__/upsert/error.test.ts @@ -1,4 +1,4 @@ -import type { KintoneRecord } from "../../../types/record"; +import type { LocalRecord } from "../../../types/record"; import { UpsertRecordsError } from "../../upsert/error"; import { buildKintoneAllRecordsError } from "../add/error.test"; import type { RecordSchema } from "../../../types/schema"; @@ -29,10 +29,11 @@ describe("UpsertRecordsError", () => { const numOfAlreadyImportedRecords = 10; const numOfProcessedRecords = 30; const errorIndex = 44; - const records: KintoneRecord[] = [...Array(numOfAllRecords).keys()].map( + const records: LocalRecord[] = [...Array(numOfAllRecords).keys()].map( (index) => ({ data: {}, metadata: { + recordIndex: index, format: { type: "csv", firstRowIndex: index + 1, diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonExistentField/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonExistentField/index.ts index 897bf5dd97..787cb24cbf 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonExistentField/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonExistentField/index.ts @@ -2,12 +2,14 @@ import type { TestPattern } from "../../index.test"; import { records } from "./records"; import { schema } from "../schema"; import { recordsOnKintone } from "../recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should throw error when non-existent field is passed as update key", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "nonExistentField", options: { diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonExistentField/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonExistentField/records.ts index a36703eaf8..1e585b68b4 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonExistentField/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonExistentField/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { singleLineText: { @@ -16,7 +16,9 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -33,6 +35,8 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonUniqueKey/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonUniqueKey/index.ts index 3b4be06901..ee7902aef8 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonUniqueKey/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonUniqueKey/index.ts @@ -2,11 +2,13 @@ import type { TestPattern } from "../../index.test"; import { records } from "./records"; import { schema } from "../schema"; import { recordsOnKintone } from "../recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should throw error when update key field is not unique", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "singleLineText_nonUnique", options: { diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonUniqueKey/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonUniqueKey/records.ts index a36703eaf8..1e585b68b4 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonUniqueKey/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNonUniqueKey/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { singleLineText: { @@ -16,7 +16,9 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -33,6 +35,8 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNumber/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNumber/index.ts index 44a0eeed41..2e8a27031d 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNumber/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNumber/index.ts @@ -3,11 +3,13 @@ import { records } from "./records"; import { schema } from "../schema"; import { expected } from "./expected"; import { recordsOnKintone } from "../recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should upsert records correctly with number", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "number", options: { diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNumber/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNumber/records.ts index a36703eaf8..1e585b68b4 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNumber/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByNumber/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { singleLineText: { @@ -16,7 +16,9 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -33,6 +35,8 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumber/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumber/index.ts index 9834cbd574..78cf0f2fe5 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumber/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumber/index.ts @@ -3,11 +3,13 @@ import { records } from "./records"; import { schema } from "./schema"; import { expected } from "./expected"; import { recordsOnKintone } from "./recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should upsert records correctly with record number", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "recordNumber", options: { diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumber/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumber/records.ts index 1203aeb20b..31f65c6e1e 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumber/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumber/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { recordNumber: { @@ -13,7 +13,9 @@ export const records: KintoneRecord[] = [ value: "1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -27,6 +29,8 @@ export const records: KintoneRecord[] = [ value: "3", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCode/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCode/index.ts index e52d9b1da8..bed36afa54 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCode/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCode/index.ts @@ -3,12 +3,14 @@ import { records } from "./records"; import { schema } from "./schema"; import { expected } from "./expected"; import { recordsOnKintone } from "./recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should upsert records correctly with record numbers on local and kintone have app code", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "recordNumber", options: { diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCode/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCode/records.ts index 77d3913ebd..79769bf587 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCode/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCode/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { recordNumber: { @@ -13,7 +13,9 @@ export const records: KintoneRecord[] = [ value: "1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -27,6 +29,8 @@ export const records: KintoneRecord[] = [ value: "3", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCodeOnKintone/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCodeOnKintone/index.ts index 32e67e881a..7bb8cae0b8 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCodeOnKintone/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCodeOnKintone/index.ts @@ -3,12 +3,14 @@ import { records } from "./records"; import { schema } from "./schema"; import { expected } from "./expected"; import { recordsOnKintone } from "./recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should upsert records correctly with record number on kintone has app code", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "recordNumber", options: { diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCodeOnKintone/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCodeOnKintone/records.ts index 1203aeb20b..31f65c6e1e 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCodeOnKintone/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithAppCodeOnKintone/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { recordNumber: { @@ -13,7 +13,9 @@ export const records: KintoneRecord[] = [ value: "1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -27,6 +29,8 @@ export const records: KintoneRecord[] = [ value: "3", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithInvalidRecordNumber/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithInvalidRecordNumber/index.ts index 25fcf34a0e..6e308e4871 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithInvalidRecordNumber/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithInvalidRecordNumber/index.ts @@ -2,12 +2,13 @@ import type { TestPattern } from "../../index.test"; import { records } from "./records"; import { schema } from "./schema"; import { recordsOnKintone } from "./recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { - description: - "should throw error because the record numbers are mixed with those with and without app code", + description: "should throw error because the record number is invalid", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "recordNumber", options: { @@ -18,8 +19,7 @@ export const pattern: TestPattern = { recordsOnKintone: recordsOnKintone, expected: { failure: { - errorMessage: - 'The "Key to Bulk Update" should not be mixed with those with and without app code', + errorMessage: 'The "Key to Bulk Update" value is invalid (Hoge-3)', }, }, }; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithInvalidRecordNumber/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithInvalidRecordNumber/records.ts index 2191330857..74439406d0 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithInvalidRecordNumber/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithInvalidRecordNumber/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { recordNumber: { @@ -13,12 +13,14 @@ export const records: KintoneRecord[] = [ value: "1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { recordNumber: { - value: "App-3", + value: "Hoge-3", }, singleLineText: { value: "value3", @@ -27,6 +29,8 @@ export const records: KintoneRecord[] = [ value: "3", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithMixedRecordNumber/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithMixedRecordNumber/index.ts index 4343e43dc5..8f9aec09d0 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithMixedRecordNumber/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithMixedRecordNumber/index.ts @@ -2,12 +2,14 @@ import type { TestPattern } from "../../index.test"; import { records } from "./records"; import { schema } from "./schema"; import { recordsOnKintone } from "./recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should throw error because the record numbers are mixed with those with and without app code", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "recordNumber", options: { @@ -18,7 +20,8 @@ export const pattern: TestPattern = { recordsOnKintone: recordsOnKintone, expected: { failure: { - errorMessage: 'The "Key to Bulk Update" value is invalid (Hoge-3)', + errorMessage: + 'The "Key to Bulk Update" should not be mixed with those with and without app code', }, }, }; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithMixedRecordNumber/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithMixedRecordNumber/records.ts index a3ce14386c..7c5ea9856e 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithMixedRecordNumber/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByRecordNumberWithMixedRecordNumber/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { recordNumber: { @@ -13,12 +13,14 @@ export const records: KintoneRecord[] = [ value: "1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { recordNumber: { - value: "Hoge-3", + value: "App-3", }, singleLineText: { value: "value3", @@ -27,6 +29,8 @@ export const records: KintoneRecord[] = [ value: "3", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertBySingleLineText/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertBySingleLineText/index.ts index 9ea321e9b8..97f029cf40 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertBySingleLineText/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertBySingleLineText/index.ts @@ -3,11 +3,13 @@ import { records } from "./records"; import { schema } from "../schema"; import { expected } from "./expected"; import { recordsOnKintone } from "../recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should upsert records correctly with single line text", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "singleLineText", options: { diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertBySingleLineText/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertBySingleLineText/records.ts index a36703eaf8..1e585b68b4 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertBySingleLineText/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertBySingleLineText/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { singleLineText: { @@ -16,7 +16,9 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -33,6 +35,8 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByUnsupportedField/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByUnsupportedField/index.ts index e24fa5083b..7ae03d6755 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByUnsupportedField/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByUnsupportedField/index.ts @@ -2,12 +2,14 @@ import type { TestPattern } from "../../index.test"; import { records } from "./records"; import { schema } from "../schema"; import { recordsOnKintone } from "../recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should throw error when unsupported field is passed as update key", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "date", options: { diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByUnsupportedField/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByUnsupportedField/records.ts index a36703eaf8..1e585b68b4 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertByUnsupportedField/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertByUnsupportedField/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { singleLineText: { @@ -16,7 +16,9 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -33,6 +35,8 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertRecordsSequentially/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertRecordsSequentially/index.ts index 545847be73..affefca36c 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertRecordsSequentially/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertRecordsSequentially/index.ts @@ -3,11 +3,13 @@ import { records } from "./records"; import { schema } from "./schema"; import { expected } from "./expected"; import { recordsOnKintone } from "./recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should upsert records with correct order", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "recordNumber", options: { diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertRecordsSequentially/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertRecordsSequentially/records.ts index 68d022e7ce..25b21f3c03 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertRecordsSequentially/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertRecordsSequentially/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { recordNumber: { @@ -13,7 +13,9 @@ export const records: KintoneRecord[] = [ value: "11", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -27,7 +29,9 @@ export const records: KintoneRecord[] = [ value: "22", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, { data: { @@ -41,7 +45,9 @@ export const records: KintoneRecord[] = [ value: "33", }, }, - metadata: { format: { type: "csv", firstRowIndex: 3, lastRowIndex: 3 } }, + metadata: { + format: { type: "csv", firstRowIndex: 3, lastRowIndex: 3 }, + }, }, { data: { @@ -55,7 +61,9 @@ export const records: KintoneRecord[] = [ value: "1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 4, lastRowIndex: 4 } }, + metadata: { + format: { type: "csv", firstRowIndex: 4, lastRowIndex: 4 }, + }, }, { data: { @@ -69,7 +77,9 @@ export const records: KintoneRecord[] = [ value: "2", }, }, - metadata: { format: { type: "csv", firstRowIndex: 5, lastRowIndex: 5 } }, + metadata: { + format: { type: "csv", firstRowIndex: 5, lastRowIndex: 5 }, + }, }, { data: { @@ -83,7 +93,9 @@ export const records: KintoneRecord[] = [ value: "44", }, }, - metadata: { format: { type: "csv", firstRowIndex: 6, lastRowIndex: 6 } }, + metadata: { + format: { type: "csv", firstRowIndex: 6, lastRowIndex: 6 }, + }, }, { data: { @@ -97,6 +109,8 @@ export const records: KintoneRecord[] = [ value: "55", }, }, - metadata: { format: { type: "csv", firstRowIndex: 7, lastRowIndex: 7 } }, + metadata: { + format: { type: "csv", firstRowIndex: 7, lastRowIndex: 7 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldFromRecord/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldFromRecord/index.ts index 13e60673fe..d8847684ef 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldFromRecord/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldFromRecord/index.ts @@ -2,12 +2,14 @@ import type { TestPattern } from "../../index.test"; import { records } from "./records"; import { schema } from "../schema"; import { recordsOnKintone } from "../recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should throw error when the field specified in schema does not exist on input record", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "number", options: { diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldFromRecord/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldFromRecord/records.ts index a36703eaf8..1e585b68b4 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldFromRecord/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldFromRecord/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { singleLineText: { @@ -16,7 +16,9 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -33,6 +35,8 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldInTableFromRecord/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldInTableFromRecord/index.ts index 35816dd0e7..fccfcb52d1 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldInTableFromRecord/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldInTableFromRecord/index.ts @@ -2,12 +2,14 @@ import type { TestPattern } from "../../index.test"; import { records } from "./records"; import { schema } from "./schema"; import { recordsOnKintone } from "./recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should throw error when the field in table specified in schema does not exist on input record", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "number", options: { diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldInTableFromRecord/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldInTableFromRecord/records.ts index 76c0c0200c..4c625060b5 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldInTableFromRecord/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingFieldInTableFromRecord/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { singleLineText: { @@ -20,7 +20,9 @@ export const records: KintoneRecord[] = [ ], }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -41,6 +43,8 @@ export const records: KintoneRecord[] = [ ], }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingKeyFromRecord/index.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingKeyFromRecord/index.ts index 06b72174ad..43296f2cba 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingKeyFromRecord/index.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingKeyFromRecord/index.ts @@ -2,12 +2,14 @@ import type { TestPattern } from "../../index.test"; import { records } from "./records"; import { schema } from "../schema"; import { recordsOnKintone } from "../recordsOnKintone"; +import { LocalRecordRepositoryMock } from "../../../../../repositories/localRecordRepositoryMock"; export const pattern: TestPattern = { description: "should throw error when update key field does not exist on input record", input: { records: records, + repository: new LocalRecordRepositoryMock(records, "csv"), schema: schema, updateKey: "singleLineText_nonExistentOnInput", options: { diff --git a/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingKeyFromRecord/records.ts b/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingKeyFromRecord/records.ts index a36703eaf8..1e585b68b4 100644 --- a/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingKeyFromRecord/records.ts +++ b/src/record/import/usecases/__tests__/upsert/fixtures/upsertWithMissingKeyFromRecord/records.ts @@ -1,6 +1,6 @@ -import type { KintoneRecord } from "../../../../../types/record"; +import type { LocalRecord } from "../../../../../types/record"; -export const records: KintoneRecord[] = [ +export const records: LocalRecord[] = [ { data: { singleLineText: { @@ -16,7 +16,9 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 } }, + metadata: { + format: { type: "csv", firstRowIndex: 1, lastRowIndex: 1 }, + }, }, { data: { @@ -33,6 +35,8 @@ export const records: KintoneRecord[] = [ value: "value1", }, }, - metadata: { format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 } }, + metadata: { + format: { type: "csv", firstRowIndex: 2, lastRowIndex: 2 }, + }, }, ]; diff --git a/src/record/import/usecases/__tests__/upsert/index.test.ts b/src/record/import/usecases/__tests__/upsert/index.test.ts index 7466f8c2b3..de940cdf1f 100644 --- a/src/record/import/usecases/__tests__/upsert/index.test.ts +++ b/src/record/import/usecases/__tests__/upsert/index.test.ts @@ -1,4 +1,4 @@ -import type { KintoneRecord } from "../../../types/record"; +import type { LocalRecord } from "../../../types/record"; import type { RecordSchema } from "../../../types/schema"; import { KintoneRestAPIClient } from "@kintone/rest-api-client"; @@ -16,14 +16,16 @@ import { pattern as upsertByNonExistentField } from "./fixtures/upsertByNonExist import { pattern as upsertWithMissingKeyFromRecord } from "./fixtures/upsertWithMissingKeyFromRecord"; import { pattern as upsertWithMissingFieldFromRecord } from "./fixtures/upsertWithMissingFieldFromRecord"; import { pattern as upsertWithMissingFieldInTableFromRecord } from "./fixtures/upsertWithMissingFieldInTableFromRecord"; -import { pattern as upsertByRecordNumberWithMixedRecordNumber } from "./fixtures/upsertByRecordNumberWithMixedRecordNumber"; -import { pattern as upsertByRecordNumberWithInvalidRecordNumber } from "./fixtures/upsertByRecordNumberWithInvalidRecordNumber"; +import { pattern as upsertByRecordNumberWithMixedRecordNumber } from "./fixtures/upsertByRecordNumberWithInvalidRecordNumber"; +import { pattern as upsertByRecordNumberWithInvalidRecordNumber } from "./fixtures/upsertByRecordNumberWithMixedRecordNumber"; import { UpsertRecordsError } from "../../upsert/error"; +import type { LocalRecordRepository } from "../../interface"; export type TestPattern = { description: string; input: { - records: KintoneRecord[]; + records: LocalRecord[]; + repository: LocalRecordRepository; schema: RecordSchema; updateKey: string; options: { @@ -107,7 +109,7 @@ describe("upsertRecords", () => { await upsertRecords( apiClient, APP_ID, - input.records, + input.repository, input.schema, input.updateKey, input.options @@ -125,7 +127,7 @@ describe("upsertRecords", () => { upsertRecords( apiClient, APP_ID, - input.records, + input.repository, input.schema, input.updateKey, input.options diff --git a/src/record/import/usecases/add.ts b/src/record/import/usecases/add.ts index 82a1f28c51..011aa04bef 100644 --- a/src/record/import/usecases/add.ts +++ b/src/record/import/usecases/add.ts @@ -1,5 +1,5 @@ import type { KintoneRestAPIClient } from "@kintone/rest-api-client"; -import type { KintoneRecord } from "../types/record"; +import type { LocalRecord } from "../types/record"; import type { KintoneRecordForParameter } from "../../../kintone/types"; import type { RecordSchema } from "../types/schema"; @@ -7,13 +7,15 @@ import { fieldProcessor, recordReducer } from "./add/record"; import { AddRecordsError } from "./add/error"; import { logger } from "../../../utils/log"; import { ProgressLogger } from "./add/progress"; +import type { LocalRecordRepository } from "./interface"; +import { chunked } from "../../../utils/iterator"; const CHUNK_SIZE = 2000; export const addRecords: ( apiClient: KintoneRestAPIClient, app: string, - records: KintoneRecord[], + recordSource: LocalRecordRepository, schema: RecordSchema, options: { attachmentsDir?: string; @@ -22,20 +24,27 @@ export const addRecords: ( ) => Promise = async ( apiClient, app, - records, + recordSource: LocalRecordRepository, schema, { attachmentsDir, skipMissingFields = true } ) => { let currentIndex = 0; - const progressLogger = new ProgressLogger(logger, records.length); + let currentRecords: LocalRecord[] = []; + const progressLogger = new ProgressLogger( + logger, + await recordSource.length() + ); try { logger.info("Starting to import records..."); - for (const [recordsNext, index] of recordReader(records)) { - currentIndex = index; + for await (const recordsByChunk of chunked( + recordSource.reader(), + CHUNK_SIZE + )) { + currentRecords = recordsByChunk; const recordsToUpload = await convertRecordsToApiRequestParameter( apiClient, app, - recordsNext, + recordsByChunk, schema, { attachmentsDir, @@ -46,19 +55,20 @@ export const addRecords: ( app, records: recordsToUpload, }); - progressLogger.update(index + recordsNext.length); + currentIndex += recordsByChunk.length; + progressLogger.update(currentIndex); } progressLogger.done(); } catch (e) { progressLogger.abort(currentIndex); - throw new AddRecordsError(e, records, currentIndex, schema); + throw new AddRecordsError(e, currentRecords, currentIndex, schema); } }; const convertRecordsToApiRequestParameter = async ( apiClient: KintoneRestAPIClient, app: string, - records: KintoneRecord[], + records: LocalRecord[], schema: RecordSchema, options: { attachmentsDir?: string; @@ -83,22 +93,3 @@ const convertRecordsToApiRequestParameter = async ( } return kintoneRecords; }; - -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/Arrow_functions#use_of_the_yield_keyword -// eslint-disable-next-line func-style -function* recordReader( - records: KintoneRecord[] -): Generator<[KintoneRecord[], number], void, undefined> { - if (records.length === 0) { - return; - } - - let index = 0; - while (index < records.length) { - const last = Math.min(index + CHUNK_SIZE - 1, records.length - 1); - - yield [records.slice(index, last + 1), index]; - - index = last + 1; - } -} diff --git a/src/record/import/usecases/add/error.ts b/src/record/import/usecases/add/error.ts index d255ca6d2e..e8b62c315b 100644 --- a/src/record/import/usecases/add/error.ts +++ b/src/record/import/usecases/add/error.ts @@ -1,6 +1,6 @@ import { KintoneAllRecordsError } from "@kintone/rest-api-client"; import { kintoneAllRecordsErrorToString } from "../../../error"; -import type { KintoneRecord } from "../../types/record"; +import type { LocalRecord } from "../../types/record"; import type { RecordSchema } from "../../types/schema"; import { ErrorParser } from "../../utils/error"; @@ -11,14 +11,14 @@ const ADD_RECORDS_LIMIT = 100; export class AddRecordsError extends Error { private readonly cause: unknown; private readonly chunkSize: number = ADD_RECORDS_LIMIT; - private readonly records: KintoneRecord[]; + private readonly records: LocalRecord[]; private readonly numOfSuccess: number; private readonly numOfTotal: number; private readonly recordSchema: RecordSchema; constructor( cause: unknown, - records: KintoneRecord[], + records: LocalRecord[], currentIndex: number, recordSchema: RecordSchema ) { diff --git a/src/record/import/usecases/add/record.ts b/src/record/import/usecases/add/record.ts index a840804069..f475d3e911 100644 --- a/src/record/import/usecases/add/record.ts +++ b/src/record/import/usecases/add/record.ts @@ -1,13 +1,13 @@ import type { KintoneRecordForParameter } from "../../../../kintone/types"; import type { KintoneRestAPIClient } from "@kintone/rest-api-client"; -import type { KintoneRecord } from "../../types/record"; +import type { LocalRecord } from "../../types/record"; import type * as Fields from "../../types/field"; import type { FieldSchema, RecordSchema } from "../../types/schema"; import path from "path"; export const recordReducer: ( - record: KintoneRecord, + record: LocalRecord, schema: RecordSchema, skipMissingFields: boolean, task: ( diff --git a/src/record/import/usecases/interface.ts b/src/record/import/usecases/interface.ts new file mode 100644 index 0000000000..354fe1d4c6 --- /dev/null +++ b/src/record/import/usecases/interface.ts @@ -0,0 +1,8 @@ +import type { LocalRecord } from "../types/record"; + +export type LocalRecordRepository = { + readonly format: string; + readonly length: () => Promise; + + readonly reader: () => AsyncGenerator; +}; diff --git a/src/record/import/usecases/upsert.ts b/src/record/import/usecases/upsert.ts index 8ed6d30675..ed453f4db5 100644 --- a/src/record/import/usecases/upsert.ts +++ b/src/record/import/usecases/upsert.ts @@ -1,5 +1,5 @@ import type { KintoneRestAPIClient } from "@kintone/rest-api-client"; -import type { KintoneRecord } from "../types/record"; +import type { LocalRecord } from "../types/record"; import type { KintoneRecordForParameter, KintoneRecordForUpdateParameter, @@ -11,13 +11,15 @@ import { UpdateKey } from "./upsert/updateKey"; import { UpsertRecordsError } from "./upsert/error"; import { logger } from "../../../utils/log"; import { ProgressLogger } from "./add/progress"; +import type { LocalRecordRepository } from "./interface"; +import { groupByKeyChunked } from "../../../utils/iterator"; const CHUNK_SIZE = 2000; export const upsertRecords = async ( apiClient: KintoneRestAPIClient, app: string, - records: KintoneRecord[], + recordSource: LocalRecordRepository, schema: RecordSchema, updateKeyCode: string, { @@ -26,7 +28,11 @@ export const upsertRecords = async ( }: { attachmentsDir?: string; skipMissingFields?: boolean } ): Promise => { let currentIndex = 0; - const progressLogger = new ProgressLogger(logger, records.length); + let currentRecords: LocalRecord[] = []; + const progressLogger = new ProgressLogger( + logger, + await recordSource.length() + ); try { logger.info("Preparing to import records..."); const updateKey = await UpdateKey.build( @@ -35,16 +41,22 @@ export const upsertRecords = async ( updateKeyCode, schema ); - updateKey.validateUpdateKeyInRecords(records); + + await updateKey.validateUpdateKeyInRecords(recordSource); logger.info("Starting to import records..."); - for (const [recordsNext, index] of recordReader(records, updateKey)) { - currentIndex = index; - if (recordsNext.type === "update") { + for await (const recordsByChunk of groupByKeyChunked( + recordSource.reader(), + (record) => (updateKey.isUpdate(record) ? "update" : "add"), + CHUNK_SIZE + )) { + currentRecords = recordsByChunk.data; + + if (recordsByChunk.key === "update") { const recordsToUpload = await convertToKintoneRecordForUpdate( apiClient, app, - recordsNext.records, + recordsByChunk.data, schema, updateKey, { attachmentsDir, skipMissingFields } @@ -57,7 +69,7 @@ export const upsertRecords = async ( const recordsToUpload = await convertToKintoneRecordForAdd( apiClient, app, - recordsNext.records, + recordsByChunk.data, schema, updateKey, { attachmentsDir, skipMissingFields } @@ -67,19 +79,20 @@ export const upsertRecords = async ( records: recordsToUpload, }); } - progressLogger.update(index + recordsNext.records.length); + currentIndex += recordsByChunk.data.length; + progressLogger.update(currentIndex); } progressLogger.done(); } catch (e) { progressLogger.abort(currentIndex); - throw new UpsertRecordsError(e, records, currentIndex, schema); + throw new UpsertRecordsError(e, currentRecords, currentIndex, schema); } }; const convertToKintoneRecordForUpdate = async ( apiClient: KintoneRestAPIClient, app: string, - records: KintoneRecord[], + records: LocalRecord[], schema: RecordSchema, updateKey: UpdateKey, options: { @@ -125,7 +138,7 @@ const convertToKintoneRecordForUpdate = async ( const convertToKintoneRecordForAdd = async ( apiClient: KintoneRestAPIClient, app: string, - records: KintoneRecord[], + records: LocalRecord[], schema: RecordSchema, updateKey: UpdateKey, options: { @@ -158,43 +171,3 @@ const convertToKintoneRecordForAdd = async ( return kintoneRecords; }; - -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/Arrow_functions#use_of_the_yield_keyword -// eslint-disable-next-line func-style -function* recordReader( - records: KintoneRecord[], - updateKey: UpdateKey -): Generator< - [{ type: "add" | "update"; records: KintoneRecord[] }, number], - void, - undefined -> { - if (records.length === 0) { - return; - } - - let index = 0; - while (index < records.length) { - let last = index; - const isUpdateCurrent = updateKey.isUpdate(records[index]); - - while ( - last + 1 < records.length && - last - index < CHUNK_SIZE - 1 && - updateKey.isUpdate(records[last + 1]) === isUpdateCurrent - ) { - last++; - } - - yield [ - { - type: isUpdateCurrent ? "update" : "add", - records: records.slice(index, last + 1), - }, - index, - ]; - - index = last + 1; - last = index; - } -} diff --git a/src/record/import/usecases/upsert/error.ts b/src/record/import/usecases/upsert/error.ts index 715aae1e35..0fc369c40c 100644 --- a/src/record/import/usecases/upsert/error.ts +++ b/src/record/import/usecases/upsert/error.ts @@ -1,7 +1,7 @@ import { KintoneAllRecordsError } from "@kintone/rest-api-client"; import { ErrorParser } from "../../utils/error"; import { kintoneAllRecordsErrorToString } from "../../../error"; -import type { KintoneRecord } from "../../types/record"; +import type { LocalRecord } from "../../types/record"; import type { RecordSchema } from "../../types/schema"; // Magic number from @kintone/rest-api-client @@ -11,14 +11,14 @@ const UPDATE_RECORDS_LIMIT = 100; export class UpsertRecordsError extends Error { private readonly cause: unknown; private readonly chunkSize: number = UPDATE_RECORDS_LIMIT; - private readonly records: KintoneRecord[]; + private readonly records: LocalRecord[]; private readonly numOfSuccess: number; private readonly numOfTotal: number; private readonly recordSchema: RecordSchema; constructor( cause: unknown, - records: KintoneRecord[], + records: LocalRecord[], currentIndex: number, recordSchema: RecordSchema ) { diff --git a/src/record/import/usecases/upsert/updateKey.ts b/src/record/import/usecases/upsert/updateKey.ts index de56129f65..ee464b2e52 100644 --- a/src/record/import/usecases/upsert/updateKey.ts +++ b/src/record/import/usecases/upsert/updateKey.ts @@ -4,7 +4,9 @@ import type { } from "@kintone/rest-api-client"; import type { FieldSchema, RecordSchema } from "../../types/schema"; -import type { KintoneRecord } from "../../types/record"; +import type { LocalRecord } from "../../types/record"; +import type { LocalRecordRepository } from "../interface"; +import { withIndex } from "../../../../utils/iterator"; type UpdateKeyField = { code: string; @@ -53,10 +55,10 @@ export class UpdateKey { getUpdateKeyField = () => this.field; - validateUpdateKeyInRecords = (records: KintoneRecord[]) => + validateUpdateKeyInRecords = (records: LocalRecordRepository) => validateUpdateKeyInRecords(this.field, this.appCode, records); - isUpdate = (record: KintoneRecord) => { + isUpdate = (record: LocalRecord) => { const updateKeyValue = this.findUpdateKeyValueFromRecord(record); return ( updateKeyValue.length > 0 && @@ -64,7 +66,7 @@ export class UpdateKey { ); }; - findUpdateKeyValueFromRecord = (record: KintoneRecord): string => { + findUpdateKeyValueFromRecord = (record: LocalRecord): string => { const fieldValue = record.data[this.field.code].value as string; if (fieldValue.length === 0) { return fieldValue; @@ -120,13 +122,15 @@ const isSupportedUpdateKeyFieldType = ( return supportedUpdateKeyFieldTypes.includes(fieldSchema.type); }; -const validateUpdateKeyInRecords = ( +const validateUpdateKeyInRecords = async ( updateKey: UpdateKeyField, appCode: string, - records: KintoneRecord[] + recordRepository: LocalRecordRepository ) => { let hasAppCodePrevious: boolean = false; - records.forEach((record, index) => { + for await (const { data: record, index } of withIndex( + recordRepository.reader() + )) { if (!(updateKey.code in record.data)) { throw new Error( `The field specified as "Key to Bulk Update" (${updateKey.code}) does not exist on the input` @@ -152,7 +156,7 @@ const validateUpdateKeyInRecords = ( } hasAppCodePrevious = _hasAppCode; } - }); + } }; const removeAppCode = (input: string, appCode: string) => { diff --git a/src/record/import/utils/__tests__/error.test.ts b/src/record/import/utils/__tests__/error.test.ts index 2cb8d32ad9..52e6ebd6fa 100644 --- a/src/record/import/utils/__tests__/error.test.ts +++ b/src/record/import/utils/__tests__/error.test.ts @@ -1,4 +1,4 @@ -import type { KintoneRecord } from "../../types/record"; +import type { LocalRecord } from "../../types/record"; import type { KintoneErrorResponse } from "@kintone/rest-api-client"; import { KintoneAllRecordsError, @@ -37,10 +37,11 @@ describe("kintoneAllRecordsErrorToString", () => { const errorIndex = 44; const errorFieldCode = "number"; const errorRowIndex = errorIndex + 2; - const records: KintoneRecord[] = [...Array(numOfAllRecords).keys()].map( + const records: LocalRecord[] = [...Array(numOfAllRecords).keys()].map( (index) => ({ data: {}, metadata: { + recordIndex: index, format: { type: "csv", firstRowIndex: index + 1, diff --git a/src/record/import/utils/error.ts b/src/record/import/utils/error.ts index e2748704ad..1d5d89c3d4 100644 --- a/src/record/import/utils/error.ts +++ b/src/record/import/utils/error.ts @@ -1,19 +1,19 @@ import type { KintoneAllRecordsErrorParser } from "../../error/types/parser"; import type { KintoneAllRecordsError } from "@kintone/rest-api-client"; -import type { KintoneRecord } from "../types/record"; +import type { LocalRecord } from "../types/record"; import type { RecordSchema } from "../types/schema"; export class ErrorParser implements KintoneAllRecordsErrorParser { private readonly error: KintoneAllRecordsError; private readonly chunkSize: number; - private readonly records: KintoneRecord[]; + private readonly records: LocalRecord[]; private readonly offset: number; private readonly recordSchema: RecordSchema; constructor( error: KintoneAllRecordsError, chunkSize: number, - records: KintoneRecord[], + records: LocalRecord[], offset: number, recordSchema: RecordSchema ) { diff --git a/src/utils/__tests__/iterator.test.ts b/src/utils/__tests__/iterator.test.ts new file mode 100644 index 0000000000..e1b7f96ef9 --- /dev/null +++ b/src/utils/__tests__/iterator.test.ts @@ -0,0 +1,186 @@ +import { + chunked, + groupByKey, + groupByKeyChunked, + iterToAsyncIter, + withIndex, + withNext, +} from "../iterator"; + +const arrayToAsyncIter = (arr: T[]): AsyncIterableIterator => + iterToAsyncIter(arr[Symbol.iterator]()); + +describe("chunked", () => { + it("can separate data source by chunk", async () => { + const source = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + const expected = [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10]]; + const actual = []; + for await (const chunk of chunked(arrayToAsyncIter(source), 3)) { + actual.push(chunk); + } + expect(actual).toStrictEqual(expected); + }); +}); + +describe("groupByKey", () => { + it("can separate date source by group", async () => { + const source = [ + { name: "banana", type: "fruit" }, + { name: "orange", type: "fruit" }, + { name: "tomato", type: "vegetables" }, + { name: "carrot", type: "vegetables" }, + { name: "asparagus", type: "vegetables" }, + { name: "beef", type: "meat" }, + { name: "cherries", type: "fruit" }, + { name: "pork", type: "meat" }, + { name: "chicken", type: "meat" }, + ]; + const expected = [ + { + key: "fruit", + data: [ + { name: "banana", type: "fruit" }, + { name: "orange", type: "fruit" }, + ], + }, + { + key: "vegetables", + data: [ + { name: "tomato", type: "vegetables" }, + { name: "carrot", type: "vegetables" }, + { name: "asparagus", type: "vegetables" }, + ], + }, + { + key: "meat", + data: [{ name: "beef", type: "meat" }], + }, + { + key: "fruit", + data: [{ name: "cherries", type: "fruit" }], + }, + { + key: "meat", + data: [ + { name: "pork", type: "meat" }, + { name: "chicken", type: "meat" }, + ], + }, + ]; + const actual = []; + for await (const chunk of groupByKey( + arrayToAsyncIter(source), + (el) => el.type + )) { + actual.push(chunk); + } + expect(actual).toStrictEqual(expected); + }); +}); + +describe("groupByKeyChunked", () => { + it("can separate date source by group and chunked", async () => { + const chunkSize = 2; + const source = [ + { name: "banana", type: "fruit" }, + { name: "orange", type: "fruit" }, + { name: "tomato", type: "vegetables" }, + { name: "carrot", type: "vegetables" }, + { name: "asparagus", type: "vegetables" }, + { name: "beef", type: "meat" }, + { name: "cherries", type: "fruit" }, + { name: "pork", type: "meat" }, + { name: "chicken", type: "meat" }, + ]; + const expected = [ + { + key: "fruit", + data: [ + { name: "banana", type: "fruit" }, + { name: "orange", type: "fruit" }, + ], + }, + { + key: "vegetables", + data: [ + { name: "tomato", type: "vegetables" }, + { name: "carrot", type: "vegetables" }, + ], + }, + { + key: "vegetables", + data: [{ name: "asparagus", type: "vegetables" }], + }, + { + key: "meat", + data: [{ name: "beef", type: "meat" }], + }, + { + key: "fruit", + data: [{ name: "cherries", type: "fruit" }], + }, + { + key: "meat", + data: [ + { name: "pork", type: "meat" }, + { name: "chicken", type: "meat" }, + ], + }, + ]; + const actual = []; + for await (const chunk of groupByKeyChunked( + arrayToAsyncIter(source), + (el) => el.type, + chunkSize + )) { + actual.push(chunk); + } + expect(actual).toStrictEqual(expected); + }); +}); + +describe("withNextIterator", () => { + it("can iterate from data source with next value", async () => { + const source = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + const expected = [ + { current: 1, next: 2 }, + { current: 2, next: 3 }, + { current: 3, next: 4 }, + { current: 4, next: 5 }, + { current: 5, next: 6 }, + { current: 6, next: 7 }, + { current: 7, next: 8 }, + { current: 8, next: 9 }, + { current: 9, next: 10 }, + { current: 10, next: undefined }, + ]; + const actual = []; + for await (const data of withNext(arrayToAsyncIter(source))) { + actual.push(data); + } + expect(actual).toStrictEqual(expected); + }); +}); + +describe("withIndexIterator", () => { + it("can iterate from data source with index", async () => { + const source = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + const expected = [ + { data: 1, index: 0 }, + { data: 2, index: 1 }, + { data: 3, index: 2 }, + { data: 4, index: 3 }, + { data: 5, index: 4 }, + { data: 6, index: 5 }, + { data: 7, index: 6 }, + { data: 8, index: 7 }, + { data: 9, index: 8 }, + { data: 10, index: 9 }, + ]; + const actual = []; + for await (const data of withIndex(arrayToAsyncIter(source))) { + actual.push(data); + } + expect(actual).toStrictEqual(expected); + }); +}); diff --git a/src/utils/file.ts b/src/utils/file.ts index ad18c44afd..09f25d4155 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -1,9 +1,24 @@ import fs from "fs"; import path from "path"; import iconv from "iconv-lite"; +import { Transform } from "stream"; export type SupportedImportEncoding = "utf8" | "sjis"; +export const openFsStreamWithEncode: ( + filePath: string, + encoding?: SupportedImportEncoding +) => NodeJS.ReadableStream = (filePath, encoding = "utf8") => { + const stream = fs.createReadStream(filePath); + const decodedStream = stream.pipe( + Transform.from(iconv.decodeStream(encoding)) + ); + stream.on("error", (e) => { + decodedStream.destroy(e); + }); + return decodedStream; +}; + export const readFile: ( filePath: string, encoding?: SupportedImportEncoding @@ -38,7 +53,7 @@ const readStream: (stream: NodeJS.ReadWriteStream) => Promise = async ( return content; }; -const extractFileFormat: (filepath: string) => string = (filepath) => { +export const extractFileFormat: (filepath: string) => string = (filepath) => { // TODO this cannot detect file format without extensions return path.extname(filepath).split(".").pop() || ""; }; diff --git a/src/utils/iterator.ts b/src/utils/iterator.ts new file mode 100644 index 0000000000..e999b36963 --- /dev/null +++ b/src/utils/iterator.ts @@ -0,0 +1,87 @@ +type Iterators = IterableIterator | Generator; +type AsyncIterators = AsyncIterableIterator | AsyncGenerator; + +// eslint-disable-next-line func-style +export async function* iterToAsyncIter( + source: Iterators +): AsyncIterators { + for (const element of source) { + yield element; + } +} + +// eslint-disable-next-line func-style +export async function* chunked( + source: AsyncIterators, + size: number +): AsyncGenerator { + let chunk = []; + for await (const element of source) { + chunk.push(element); + if (chunk.length >= size) { + yield chunk; + chunk = []; + } + } + if (chunk.length > 0) { + yield chunk; + } +} + +// eslint-disable-next-line func-style +export async function* groupByKey( + source: AsyncIterableIterator | AsyncGenerator, + keyFn: (element: T) => K +): AsyncGenerator<{ key: K; data: T[] }, void, undefined> { + let array = []; + for await (const { current, next } of withNext(source)) { + array.push(current); + const currentKey = keyFn(current); + if (next === undefined || currentKey !== keyFn(next)) { + yield { key: currentKey, data: array }; + array = []; + } + } +} + +// eslint-disable-next-line func-style +export async function* groupByKeyChunked( + source: AsyncIterators, + keyFn: (element: T) => K, + size: number +): AsyncGenerator<{ key: K; data: T[] }, void, undefined> { + for await (const { key, data } of groupByKey(source, keyFn)) { + for await (const chunk of chunked( + iterToAsyncIter(data[Symbol.iterator]()), + size + )) { + yield { key, data: chunk }; + } + } +} + +// eslint-disable-next-line func-style +export async function* withNext( + source: AsyncIterators +): AsyncGenerator<{ current: T; next?: T }> { + let { value: prev, done } = await source.next(); + if (done) { + return; + } + for await (const value of source) { + yield { current: prev, next: value }; + prev = value; + } + yield { current: prev, next: undefined }; +} + +// eslint-disable-next-line func-style +export async function* withIndex( + source: AsyncIterators +): AsyncGenerator<{ data: T; index: number }> { + let index = 0; + for await (const value of source) { + yield { data: value, index }; + index++; + } +} diff --git a/src/utils/log.ts b/src/utils/log.ts index b651597016..3d49f9dda4 100644 --- a/src/utils/log.ts +++ b/src/utils/log.ts @@ -1,8 +1,10 @@ import { AddRecordsError } from "../record/import/usecases/add/error"; import { UpsertRecordsError } from "../record/import/usecases/upsert/error"; -import { ParserError } from "../record/import/parsers/error"; +import { ParserError } from "../record/import/repositories/parsers/error"; +import { RepositoryError } from "../record/import/repositories/error"; import { DeleteAllRecordsError } from "../record/delete/usecases/deleteAll/error"; import { DeleteSpecifiedRecordsError } from "../record/delete/usecases/deleteByRecordNumber/error"; + import chalk from "chalk"; const currentISOString = () => new Date().toISOString(); @@ -51,6 +53,8 @@ const parseErrorMessage = (error: unknown): string => { return error.toString(); } else if (error instanceof UpsertRecordsError) { return error.toString(); + } else if (error instanceof RepositoryError) { + return error.toString(); } else if (error instanceof DeleteAllRecordsError) { return error.toString(); } else if (error instanceof DeleteSpecifiedRecordsError) {