From 62b652a8f44e8aab52bc6d20385ca980b93e2cf2 Mon Sep 17 00:00:00 2001 From: Gerben Jongerius Date: Sun, 31 Mar 2024 12:37:35 +0200 Subject: [PATCH 1/8] Extract dedicated modules for importers. Allowing multiple importer types in the future. --- settings.gradle.kts | 10 +- .../transaction-importer-api/build.gradle.kts | 5 + .../finance/importer/ImportProvider.java | 14 ++ .../importer/api/ImporterConfiguration.java | 4 + .../importer/api/TransactionConsumer.java | 6 + .../finance/importer/api/TransactionDTO.java | 31 ++++ .../transaction-importer-csv/build.gradle.kts | 17 ++ .../importer/csv/CSVConfiguration.java | 25 +++ .../importer/csv/CSVImportProvider.java | 124 +++++++++++++++ .../finance/importer/csv/ColumnRole.java | 33 ++++ .../csv/TransactionTypeIndicator.java | 12 ++ .../importer/csv/serde/ColumnRoleSerde.java | 23 +++ .../importer/csv/CSVImportProviderTest.java | 147 ++++++++++++++++++ .../configuration/MockitoConfiguration.java | 69 ++++++++ .../resources/configuration/valid-config.json | 22 +++ .../resources/csv-files/single-deposit.csv | 2 + .../resources/csv-files/single-withdrawal.csv | 2 + .../src/test/resources/logback.xml | 16 ++ 18 files changed, 561 insertions(+), 1 deletion(-) create mode 100644 transaction-importer/transaction-importer-api/build.gradle.kts create mode 100644 transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImportProvider.java create mode 100644 transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/ImporterConfiguration.java create mode 100644 transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionConsumer.java create mode 100644 transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionDTO.java create mode 100644 transaction-importer/transaction-importer-csv/build.gradle.kts create mode 100644 transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVConfiguration.java create mode 100644 transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVImportProvider.java create mode 100644 transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/ColumnRole.java create mode 100644 transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/TransactionTypeIndicator.java create mode 100644 transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/serde/ColumnRoleSerde.java create mode 100644 transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java create mode 100644 transaction-importer/transaction-importer-csv/src/test/java/org/mockito/configuration/MockitoConfiguration.java create mode 100644 transaction-importer/transaction-importer-csv/src/test/resources/configuration/valid-config.json create mode 100644 transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-deposit.csv create mode 100644 transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-withdrawal.csv create mode 100644 transaction-importer/transaction-importer-csv/src/test/resources/logback.xml diff --git a/settings.gradle.kts b/settings.gradle.kts index e4876ce3..24d1a72e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -45,4 +45,12 @@ dependencyResolutionManagement { } } -include("core", "domain", "rule-engine", "bpmn-process", "jpa-repository", "fintrack-api") +include( + "core", + "domain", + "transaction-importer:transaction-importer-api", + "transaction-importer:transaction-importer-csv", + "rule-engine", + "bpmn-process", + "jpa-repository", + "fintrack-api") diff --git a/transaction-importer/transaction-importer-api/build.gradle.kts b/transaction-importer/transaction-importer-api/build.gradle.kts new file mode 100644 index 00000000..c19689c0 --- /dev/null +++ b/transaction-importer/transaction-importer-api/build.gradle.kts @@ -0,0 +1,5 @@ +dependencies { + implementation(mn.micronaut.serde.jackson) + implementation(project(":core")) + implementation(project(":domain")) +} \ No newline at end of file diff --git a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImportProvider.java b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImportProvider.java new file mode 100644 index 00000000..1ae9cce5 --- /dev/null +++ b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImportProvider.java @@ -0,0 +1,14 @@ +package com.jongsoft.finance.importer; + +import com.jongsoft.finance.domain.importer.BatchImport; +import com.jongsoft.finance.domain.importer.BatchImportConfig; +import com.jongsoft.finance.importer.api.ImporterConfiguration; +import com.jongsoft.finance.importer.api.TransactionConsumer; + +public interface ImportProvider { + + void readTransactions(TransactionConsumer consumer, T updatedConfiguration, BatchImport importJob); + + T loadConfiguration(BatchImportConfig batchImportConfig); + +} diff --git a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/ImporterConfiguration.java b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/ImporterConfiguration.java new file mode 100644 index 00000000..badd7d99 --- /dev/null +++ b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/ImporterConfiguration.java @@ -0,0 +1,4 @@ +package com.jongsoft.finance.importer.api; + +public interface ImporterConfiguration { +} diff --git a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionConsumer.java b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionConsumer.java new file mode 100644 index 00000000..a42f4c15 --- /dev/null +++ b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionConsumer.java @@ -0,0 +1,6 @@ +package com.jongsoft.finance.importer.api; + +import java.util.function.Consumer; + +public interface TransactionConsumer extends Consumer { +} diff --git a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionDTO.java b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionDTO.java new file mode 100644 index 00000000..c99a2ab8 --- /dev/null +++ b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionDTO.java @@ -0,0 +1,31 @@ +package com.jongsoft.finance.importer.api; + +import com.jongsoft.finance.core.TransactionType; +import io.micronaut.serde.annotation.Serdeable; + +import java.time.LocalDate; + +@Serdeable +public record TransactionDTO( + // The amount of the transaction + double amount, + // The type of the transaction + TransactionType type, + // The description of the transaction + String description, + // The date of the transaction + LocalDate transactionDate, + // The date the transaction starts to accrue interest + LocalDate interestDate, + // The date the transaction was booked + LocalDate bookDate, + // The IBAN of the opposing account + String opposingIBAN, + // The name of the opposing account + String opposingName) { + + @Override + public String toString() { + return "Transfer of " + amount + " to " + opposingName + " (" + opposingIBAN + ") on " + transactionDate; + } +} diff --git a/transaction-importer/transaction-importer-csv/build.gradle.kts b/transaction-importer/transaction-importer-csv/build.gradle.kts new file mode 100644 index 00000000..c60de5ce --- /dev/null +++ b/transaction-importer/transaction-importer-csv/build.gradle.kts @@ -0,0 +1,17 @@ + +micronaut { + testRuntime("junit5") +} + +dependencies { + implementation(libs.csv) + implementation(libs.lang) + implementation(mn.micronaut.serde.jackson) + + implementation(project(":transaction-importer:transaction-importer-api")) + implementation(project(":core")) + implementation(project(":domain")) + + testImplementation(libs.bundles.junit) + testRuntimeOnly(mn.logback.classic) +} \ No newline at end of file diff --git a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVConfiguration.java b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVConfiguration.java new file mode 100644 index 00000000..4fb7bf5e --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVConfiguration.java @@ -0,0 +1,25 @@ +package com.jongsoft.finance.importer.csv; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.jongsoft.finance.importer.api.ImporterConfiguration; +import io.micronaut.serde.annotation.Serdeable; + +import java.util.List; + +@Serdeable +public record CSVConfiguration( + @JsonProperty("has-headers") + boolean headers, + @JsonProperty("apply-rules") + boolean applyRules, + @JsonProperty("generate-accounts") + boolean generateAccounts, + @JsonProperty("date-format") + String dateFormat, + @JsonProperty("delimiter") + char delimiter, + @JsonProperty("custom-indicator") + TransactionTypeIndicator transactionTypeIndicator, + @JsonProperty("column-roles") + List columnRoles) implements ImporterConfiguration { +} diff --git a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVImportProvider.java b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVImportProvider.java new file mode 100644 index 00000000..f750735c --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVImportProvider.java @@ -0,0 +1,124 @@ +package com.jongsoft.finance.importer.csv; + +import com.jongsoft.finance.StorageService; +import com.jongsoft.finance.core.TransactionType; +import com.jongsoft.finance.domain.importer.BatchImport; +import com.jongsoft.finance.domain.importer.BatchImportConfig; +import com.jongsoft.finance.importer.ImportProvider; +import com.jongsoft.finance.importer.api.TransactionConsumer; +import com.jongsoft.finance.importer.api.TransactionDTO; +import com.jongsoft.lang.Control; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.exceptions.CsvValidationException; +import io.micronaut.serde.ObjectMapper; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.function.Function; + +@Singleton +public class CSVImportProvider implements ImportProvider { + private final Logger logger = LoggerFactory.getLogger(CSVImportProvider.class); + + private final StorageService storageService; + private final ObjectMapper objectMapper; + + @Inject + public CSVImportProvider(StorageService storageService, ObjectMapper objectMapper) { + this.storageService = storageService; + this.objectMapper = objectMapper; + } + + @Override + public void readTransactions(TransactionConsumer consumer, CSVConfiguration configuration, BatchImport importJob) { + logger.info("Reading transactions from CSV file: {}", importJob.getSlug()); + + try { + var inputStream = storageService.read(importJob.getFileCode()) + .map(ByteArrayInputStream::new) + .map(InputStreamReader::new) + .getOrThrow(() -> new IllegalStateException("Failed to read CSV file: " + importJob.getFileCode())); + + try (var reader = new CSVReaderBuilder(inputStream) + .withCSVParser(new CSVParserBuilder() + .withSeparator(configuration.delimiter()) + .build()) + .build()) { + + if (configuration.headers()) { + logger.debug("CSV file has headers, skipping first line"); + reader.skip(1); + } + + String[] line; + while ((line = reader.readNext()) != null) { + if (line.length != configuration.columnRoles().size()) { + logger.warn("Skipping line, columns found {} but expected is {}: {}", line.length, configuration.columnRoles().size(), line); + continue; + } + + consumer.accept(readLine(line, configuration)); + } + } + } catch (IOException | CsvValidationException e) { + logger.warn("Failed to read CSV file: {}", importJob.getFileCode(), e); + } + } + + @Override + public CSVConfiguration loadConfiguration(BatchImportConfig batchImportConfig) { + logger.debug("Loading CSV configuration from disk: {}", batchImportConfig.getFileCode()); + + try { + var jsonBytes = storageService.read(batchImportConfig.getFileCode()); + if (jsonBytes.isPresent()) { + return objectMapper.readValue(jsonBytes.get(), CSVConfiguration.class); + } + + logger.warn("No CSV configuration found on disk: {}", batchImportConfig.getFileCode()); + throw new IllegalStateException("No CSV configuration found on disk: " + batchImportConfig.getFileCode()); + } catch (IOException e) { + logger.warn("Could not load CSV configuration from disk: {}", batchImportConfig.getFileCode(), e); + throw new IllegalStateException("Failed to load CSV configuration from disk: " + batchImportConfig.getFileCode()); + } + } + + private TransactionDTO readLine(String[] line, CSVConfiguration configuration) { + Function columnLocator = (role) -> Control.Try(() -> line[configuration.columnRoles().indexOf(role)]).recover(x -> null).get(); + Function parseDate = (date) -> date != null ? LocalDate.parse(date, DateTimeFormatter.ofPattern(configuration.dateFormat())) : null; + Function parseAmount = (amount) -> Double.parseDouble(amount.replace(',', '.')); + + var amount = parseAmount.apply(columnLocator.apply(ColumnRole.AMOUNT)); + var type = Control.Option(columnLocator.apply(ColumnRole.CUSTOM_INDICATOR)) + .map(indicator -> { + if (indicator.equalsIgnoreCase(configuration.transactionTypeIndicator().credit())) { + return TransactionType.CREDIT; + } else if (indicator.equalsIgnoreCase(configuration.transactionTypeIndicator().deposit())) { + return TransactionType.DEBIT; + } + + return null; + }) + .getOrSupply(() -> amount >= 0 ? TransactionType.DEBIT : TransactionType.CREDIT); + + logger.trace("Reading single transaction on {}: amount={}, type={}", parseDate.apply(columnLocator.apply(ColumnRole.DATE)), amount, type); + + return new TransactionDTO( + amount, + type, + columnLocator.apply(ColumnRole.DESCRIPTION), + parseDate.apply(columnLocator.apply(ColumnRole.DATE)), + parseDate.apply(columnLocator.apply(ColumnRole.INTEREST_DATE)), + parseDate.apply(columnLocator.apply(ColumnRole.BOOK_DATE)), + columnLocator.apply(ColumnRole.OPPOSING_IBAN), + columnLocator.apply(ColumnRole.OPPOSING_NAME)); + } +} diff --git a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/ColumnRole.java b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/ColumnRole.java new file mode 100644 index 00000000..21b1150e --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/ColumnRole.java @@ -0,0 +1,33 @@ +package com.jongsoft.finance.importer.csv; + +public enum ColumnRole { + IGNORE("_ignore"), + DATE("transaction-date"), + BOOK_DATE("booking-date"), + INTEREST_DATE("interest-date"), + OPPOSING_NAME("opposing-name"), + OPPOSING_IBAN("opposing-iban"), + ACCOUNT_IBAN("account-iban"), + AMOUNT("amount"), + CUSTOM_INDICATOR("custom-indicator"), + DESCRIPTION("description"); + + private final String label; + + ColumnRole(String label) { + this.label = label; + } + + public String getLabel() { + return label; + } + + public static ColumnRole value(String source) { + for (ColumnRole role : values()) { + if (role.label.equalsIgnoreCase(source)) { + return role; + } + } + throw new IllegalStateException("No mapping role found for " + source); + } +} diff --git a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/TransactionTypeIndicator.java b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/TransactionTypeIndicator.java new file mode 100644 index 00000000..ad6e9bf6 --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/TransactionTypeIndicator.java @@ -0,0 +1,12 @@ +package com.jongsoft.finance.importer.csv; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.micronaut.serde.annotation.Serdeable; + +@Serdeable +public record TransactionTypeIndicator( + @JsonProperty("deposit") + String deposit, + @JsonProperty("credit") + String credit) { +} diff --git a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/serde/ColumnRoleSerde.java b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/serde/ColumnRoleSerde.java new file mode 100644 index 00000000..e8065d02 --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/serde/ColumnRoleSerde.java @@ -0,0 +1,23 @@ +package com.jongsoft.finance.importer.csv.serde; + +import com.jongsoft.finance.importer.csv.ColumnRole; +import io.micronaut.core.type.Argument; +import io.micronaut.serde.Decoder; +import io.micronaut.serde.Encoder; +import io.micronaut.serde.Serde; +import jakarta.inject.Singleton; + +import java.io.IOException; + +@Singleton +class ColumnRoleSerde implements Serde { + @Override + public ColumnRole deserialize(Decoder decoder, DecoderContext context, Argument type) throws IOException { + return ColumnRole.value(decoder.decodeString()); + } + + @Override + public void serialize(Encoder encoder, EncoderContext context, Argument type, ColumnRole value) throws IOException { + encoder.encodeString(value.getLabel()); + } +} diff --git a/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java b/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java new file mode 100644 index 00000000..7d74ad2b --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java @@ -0,0 +1,147 @@ +package com.jongsoft.finance.importer.csv; + +import com.jongsoft.finance.StorageService; +import com.jongsoft.finance.core.TransactionType; +import com.jongsoft.finance.domain.importer.BatchImport; +import com.jongsoft.finance.domain.importer.BatchImportConfig; +import com.jongsoft.finance.importer.api.TransactionConsumer; +import com.jongsoft.finance.importer.api.TransactionDTO; +import com.jongsoft.lang.Control; +import io.micronaut.serde.ObjectMapper; +import io.micronaut.test.annotation.MockBean; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import jakarta.inject.Inject; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.List; + + +@MicronautTest +class CSVImportProviderTest { + + @Inject + private CSVImportProvider csvImportProvider; + + @Inject + private ObjectMapper objectMapper; + + @Inject + private StorageService storageService; + + @MockBean + StorageService storageService() { + return Mockito.mock(StorageService.class); + } + + @Test + @DisplayName("Load configuration file is missing") + void loadConfiguration_fileMissing() { + Assertions.assertThatException() + .describedAs("No CSV configuration found on disk: my-secret-files") + .isThrownBy(() -> csvImportProvider.loadConfiguration(createBatchImportConfig())) + .isInstanceOf(IllegalStateException.class) + .withMessage("No CSV configuration found on disk: my-secret-files"); + } + + @Test + @DisplayName("Load configuration file that is valid") + void loadConfiguration() throws IOException { + Mockito.when(storageService.read("my-secret-files")) + .thenReturn(Control.Option( + getClass().getResourceAsStream("/configuration/valid-config.json") + .readAllBytes())); + + var configuration = csvImportProvider.loadConfiguration(createBatchImportConfig()); + + Assertions.assertThat(configuration) + .isNotNull() + .extracting(CSVConfiguration::delimiter, CSVConfiguration::headers, CSVConfiguration::applyRules, CSVConfiguration::generateAccounts, CSVConfiguration::dateFormat, CSVConfiguration::transactionTypeIndicator, CSVConfiguration::columnRoles) + .isEqualTo(List.of( + ',', + true, + true, + false, + "yyyyMMdd", + new TransactionTypeIndicator("Bij", "Af"), + List.of( + ColumnRole.DATE, + ColumnRole.OPPOSING_NAME, + ColumnRole.ACCOUNT_IBAN, + ColumnRole.OPPOSING_IBAN, + ColumnRole.IGNORE, + ColumnRole.CUSTOM_INDICATOR, + ColumnRole.AMOUNT, + ColumnRole.IGNORE, + ColumnRole.DESCRIPTION))); + } + + @Test + @DisplayName("Read transactions with single deposit") + void readTransactions_deposit() throws IOException { + Mockito.when(storageService.read("my-secret-import-files")) + .thenReturn(Control.Option( + getClass().getResourceAsStream("/csv-files/single-deposit.csv") + .readAllBytes())); + + var consumer = Mockito.mock(TransactionConsumer.class); + + csvImportProvider.readTransactions(consumer, createCSVConfiguration(), createBatchImport()); + + var transactionCaptor = ArgumentCaptor.forClass(TransactionDTO.class); + Mockito.verify(consumer, Mockito.times(1)).accept(transactionCaptor.capture()); + + var transaction = transactionCaptor.getValue(); + Assertions.assertThat(transaction.amount()).isEqualTo(14.19); + Assertions.assertThat(transaction.opposingIBAN()).isEqualTo("NL69INGB0123454789"); + Assertions.assertThat(transaction.opposingName()).isEqualTo("Janssen PA"); + Assertions.assertThat(transaction.transactionDate()).isEqualTo("2016-05-31"); + Assertions.assertThat(transaction.type()).isEqualTo(TransactionType.DEBIT); + } + + @Test + @DisplayName("Read transactions with single withdrawal") + void readTransactions_withdrawal() throws IOException { + Mockito.when(storageService.read("my-secret-import-files")) + .thenReturn(Control.Option( + getClass().getResourceAsStream("/csv-files/single-withdrawal.csv") + .readAllBytes())); + + var consumer = Mockito.mock(TransactionConsumer.class); + + csvImportProvider.readTransactions(consumer, createCSVConfiguration(), createBatchImport()); + + var transactionCaptor = ArgumentCaptor.forClass(TransactionDTO.class); + Mockito.verify(consumer, Mockito.times(1)).accept(transactionCaptor.capture()); + + var transaction = transactionCaptor.getValue(); + Assertions.assertThat(transaction.amount()).isEqualTo(283.90); + Assertions.assertThat(transaction.opposingIBAN()).isEqualTo("NL71INGB0009876543"); + Assertions.assertThat(transaction.opposingName()).isEqualTo("MW GA Pieterse"); + Assertions.assertThat(transaction.transactionDate()).isEqualTo("2016-05-31"); + Assertions.assertThat(transaction.type()).isEqualTo(TransactionType.CREDIT); + } + + private BatchImport createBatchImport() { + return BatchImport.builder() + .fileCode("my-secret-import-files") + .slug("this-is-a-import-slug") + .build(); + } + + private CSVConfiguration createCSVConfiguration() throws IOException { + return objectMapper.readValue( + getClass().getResourceAsStream("/configuration/valid-config.json"), + CSVConfiguration.class); + } + + private BatchImportConfig createBatchImportConfig() { + return BatchImportConfig.builder() + .fileCode("my-secret-files") + .build(); + } +} \ No newline at end of file diff --git a/transaction-importer/transaction-importer-csv/src/test/java/org/mockito/configuration/MockitoConfiguration.java b/transaction-importer/transaction-importer-csv/src/test/java/org/mockito/configuration/MockitoConfiguration.java new file mode 100644 index 00000000..65d1702c --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/test/java/org/mockito/configuration/MockitoConfiguration.java @@ -0,0 +1,69 @@ +package org.mockito.configuration; + +import com.jongsoft.finance.ResultPage; +import com.jongsoft.lang.Collections; +import com.jongsoft.lang.Control; +import com.jongsoft.lang.collection.Sequence; +import com.jongsoft.lang.control.Optional; +import org.mockito.configuration.IMockitoConfiguration; +import org.mockito.internal.stubbing.defaultanswers.ReturnsEmptyValues; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** + * This class is used to configure Mockito's default behavior. + * It implements the IMockitoConfiguration interface. + */ +@SuppressWarnings("unused") +public class MockitoConfiguration implements IMockitoConfiguration { + + /** + * This method is used to provide a default answer for all methods of a mock that are not stubbed. + * It overrides the getDefaultAnswer method from the IMockitoConfiguration interface. + * If the return type of the method is assignable from Optional, it returns Control.Option(). + * Otherwise, it uses the super class's answer method to return default values. + * + * @return An Answer that provides the default answer for unstubbed methods. + */ + @Override + public Answer getDefaultAnswer() { + return new ReturnsEmptyValues() { + @Override + public Object answer(InvocationOnMock invocation) { + if (Optional.class.isAssignableFrom(invocation.getMethod().getReturnType())) { + return Control.Option(); + } + if (Sequence.class.isAssignableFrom(invocation.getMethod().getReturnType())) { + return Collections.List(); + } + if (ResultPage.class.isAssignableFrom(invocation.getMethod().getReturnType())) { + return ResultPage.empty(); + } + return super.answer(invocation); + } + + }; + } + + /** + * This method is used to determine whether Mockito should clean the stack trace. + * It overrides the cleansStackTrace method from the IMockitoConfiguration interface. + * + * @return A boolean value indicating whether Mockito should clean the stack trace. It always returns false. + */ + @Override + public boolean cleansStackTrace() { + return false; + } + + /** + * This method is used to determine whether Mockito should enable the class cache. + * It overrides the enableClassCache method from the IMockitoConfiguration interface. + * + * @return A boolean value indicating whether Mockito should enable the class cache. It always returns false. + */ + @Override + public boolean enableClassCache() { + return false; + } +} \ No newline at end of file diff --git a/transaction-importer/transaction-importer-csv/src/test/resources/configuration/valid-config.json b/transaction-importer/transaction-importer-csv/src/test/resources/configuration/valid-config.json new file mode 100644 index 00000000..f9ade91f --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/test/resources/configuration/valid-config.json @@ -0,0 +1,22 @@ +{ + "has-headers": true, + "apply-rules": true, + "generate-accounts": false, + "date-format": "yyyyMMdd", + "delimiter": ",", + "column-roles": [ + "transaction-date", + "opposing-name", + "account-iban", + "opposing-iban", + "_ignore", + "custom-indicator", + "amount", + "_ignore", + "description" + ], + "custom-indicator": { + "deposit": "Bij", + "credit": "Af" + } +} diff --git a/transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-deposit.csv b/transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-deposit.csv new file mode 100644 index 00000000..1caf3629 --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-deposit.csv @@ -0,0 +1,2 @@ +Datum,Naam/Omschrijving,Rekening,Tegenrekening,Code,Af/Bij,Bedrag,Valuta,Mutatiesoort +20160531,Janssen PA,NL20INGB0001234567,NL69INGB0123454789,GT,Bij,"14,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399 \ No newline at end of file diff --git a/transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-withdrawal.csv b/transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-withdrawal.csv new file mode 100644 index 00000000..14a80150 --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-withdrawal.csv @@ -0,0 +1,2 @@ +Datum,Naam/Omschrijving,Rekening,Tegenrekening,Code,Af/Bij,Bedrag +20160531,MW GA Pieterse,NL20INGB0001234567,NL71INGB0009876543,GT,Af,"283,90",Internetbankieren,Naam: Mw G A Pieterse Omschrijving: inzake bestelling IBAN: NL71INGB0009876543 \ No newline at end of file diff --git a/transaction-importer/transaction-importer-csv/src/test/resources/logback.xml b/transaction-importer/transaction-importer-csv/src/test/resources/logback.xml new file mode 100644 index 00000000..f7605426 --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/test/resources/logback.xml @@ -0,0 +1,16 @@ + + + + false + + + %cyan(%d{HH:mm:ss.SSS}) %magenta[%-36X{correlationId}] %highlight(%-5level) %gray([%thread]) %magenta(%logger{36}) - %msg%n + + + + + + + + From b224c47f3293eb52a7de28800bfce1270f985371 Mon Sep 17 00:00:00 2001 From: Gerben Jongerius Date: Sun, 31 Mar 2024 12:39:17 +0200 Subject: [PATCH 2/8] Remove license from the readme --- README.md | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/README.md b/README.md index 1f934fc9..f043fd3b 100644 --- a/README.md +++ b/README.md @@ -48,20 +48,3 @@ API documentation use the url: http://localhost:8080/spec/index.html -## License -Copyright 2024 Jong Soft Development - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and -associated documentation files (the "Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial -portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT -LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE -OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From 1804b53c2e32bdc42c128e492cd031e478b70860 Mon Sep 17 00:00:00 2001 From: Gerben Jongerius Date: Mon, 1 Apr 2024 16:59:34 +0200 Subject: [PATCH 3/8] Update the importer process to use the new setup with the importer-api --- bpmn-process/build.gradle.kts | 2 + .../bpmn/ProcessEngineConfiguration.java | 4 + .../bpmn/camunda/JsonRecordSerializer.java | 72 ++++++ .../delegate/importer/CSVReaderDelegate.java | 205 ------------------ .../ExtractAccountDetailsDelegate.java | 68 +++--- .../importer/LoadImporterConfiguration.java | 35 +-- .../importer/ReadTransactionLogDelegate.java | 64 +++--- .../finance/serialized/ImportConfigJson.java | 85 -------- .../finance/serialized/ImportJobSettings.java | 16 ++ .../serialized/serde/MappingRoleSerde.java | 31 --- .../bpmn/transaction/import-job.bpmn | 9 +- .../jongsoft/finance/bpmn/ImportJobIT.java | 52 +++-- .../resources/import-test/import-test.csv | 10 +- fintrack-api/build.gradle.kts | 2 + ...ortProvider.java => ImporterProvider.java} | 6 +- .../importer/api/ImporterConfiguration.java | 3 + .../importer/csv/CSVImportProvider.java | 26 ++- 17 files changed, 248 insertions(+), 442 deletions(-) create mode 100644 bpmn-process/src/main/java/com/jongsoft/finance/bpmn/camunda/JsonRecordSerializer.java delete mode 100644 bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/CSVReaderDelegate.java delete mode 100644 bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportConfigJson.java create mode 100644 bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportJobSettings.java delete mode 100644 bpmn-process/src/main/java/com/jongsoft/finance/serialized/serde/MappingRoleSerde.java rename transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/{ImportProvider.java => ImporterProvider.java} (57%) diff --git a/bpmn-process/build.gradle.kts b/bpmn-process/build.gradle.kts index 0d202b1e..74373b74 100644 --- a/bpmn-process/build.gradle.kts +++ b/bpmn-process/build.gradle.kts @@ -23,6 +23,7 @@ dependencies { implementation(project(":core")) implementation(project(":domain")) implementation(project(":rule-engine")) + implementation(project(":transaction-importer:transaction-importer-api")) // needed for the testing of the application runtimeOnly(mn.h2) @@ -30,4 +31,5 @@ dependencies { testRuntimeOnly(mn.logback.classic) testImplementation(mn.micronaut.test.junit5) testImplementation(libs.bundles.junit) + testRuntimeOnly(project(":transaction-importer:transaction-importer-csv")) } diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java index 6f06d9aa..87a54964 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java @@ -2,10 +2,12 @@ import com.jongsoft.finance.bpmn.camunda.*; import com.jongsoft.finance.core.DataSourceMigration; +import com.jongsoft.finance.serialized.ImportJobSettings; import io.micronaut.context.ApplicationContext; import io.micronaut.context.annotation.Context; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; +import io.micronaut.serde.ObjectMapper; import lombok.extern.slf4j.Slf4j; import org.camunda.bpm.engine.HistoryService; import org.camunda.bpm.engine.ProcessEngine; @@ -54,6 +56,8 @@ public ProcessEngine processEngine() throws IOException { configuration.setHistoryCleanupBatchWindowEndTime("03:00"); configuration.setHistoryTimeToLive("P1D"); configuration.setResolverFactories(List.of(new MicronautBeanResolver(applicationContext))); + configuration.setCustomPreVariableSerializers(List.of( + new JsonRecordSerializer<>(applicationContext.getBean(ObjectMapper.class), ImportJobSettings.class))); var processEngine = configuration.buildProcessEngine(); log.info("Created camunda process engine"); diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/camunda/JsonRecordSerializer.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/camunda/JsonRecordSerializer.java new file mode 100644 index 00000000..8e18fd65 --- /dev/null +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/camunda/JsonRecordSerializer.java @@ -0,0 +1,72 @@ +package com.jongsoft.finance.bpmn.camunda; + +import io.micronaut.serde.ObjectMapper; +import org.camunda.bpm.engine.impl.variable.serializer.AbstractTypedValueSerializer; +import org.camunda.bpm.engine.impl.variable.serializer.ValueFields; +import org.camunda.bpm.engine.variable.Variables; +import org.camunda.bpm.engine.variable.impl.type.ObjectTypeImpl; +import org.camunda.bpm.engine.variable.impl.value.UntypedValueImpl; +import org.camunda.bpm.engine.variable.value.ObjectValue; +import org.camunda.bpm.engine.variable.value.TypedValue; + +import java.io.IOException; + +public class JsonRecordSerializer extends AbstractTypedValueSerializer { + + final ObjectMapper objectMapper; + final Class supportedClass; + + public JsonRecordSerializer(ObjectMapper objectMapper, Class supportedClass) { + super(new ObjectTypeImpl()); + this.objectMapper = objectMapper; + this.supportedClass = supportedClass; + } + + @Override + public String getName() { + return "record-json"; + } + + @Override + public String getSerializationDataformat() { + return getName(); + } + + @Override + public TypedValue convertToTypedValue(UntypedValueImpl untypedValue) { + var importJobSettings = (Record) untypedValue.getValue(); + String jsonString; + try { + jsonString = objectMapper.writeValueAsString(importJobSettings); + } catch (IOException e) { + throw new RuntimeException("Could not serialize ImportJobSettings", e); + } + return Variables.serializedObjectValue(jsonString) + .serializationDataFormat(getName()) + .create(); + } + + @Override + public void writeValue(TypedValue typedValue, ValueFields valueFields) { + ObjectValue objectValue = (ObjectValue) typedValue; + valueFields.setByteArrayValue(objectValue.getValueSerialized().getBytes()); + } + + @Override + public TypedValue readValue(ValueFields valueFields, boolean b, boolean b1) { + try { + return Variables.objectValue(objectMapper.readValue( + new String(valueFields.getByteArrayValue()), + supportedClass)) + .serializationDataFormat(getName()) + .create(); + } catch (IOException e) { + throw new RuntimeException("Could not deserialize ImportJobSettings", e); + } + } + + @Override + protected boolean canWriteValue(TypedValue typedValue) { + return supportedClass.isAssignableFrom(typedValue.getValue().getClass()); + } +} diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/CSVReaderDelegate.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/CSVReaderDelegate.java deleted file mode 100644 index a4a4f404..00000000 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/CSVReaderDelegate.java +++ /dev/null @@ -1,205 +0,0 @@ -package com.jongsoft.finance.bpmn.delegate.importer; - -import com.jongsoft.finance.ProcessMapper; -import com.jongsoft.finance.StorageService; -import com.jongsoft.finance.domain.transaction.Transaction; -import com.jongsoft.finance.providers.ImportProvider; -import com.jongsoft.finance.serialized.ImportConfigJson; -import com.opencsv.CSVParserBuilder; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvValidationException; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.camunda.bpm.engine.delegate.DelegateExecution; -import org.camunda.bpm.engine.delegate.JavaDelegate; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.EnumMap; -import java.util.Optional; - -@Slf4j -abstract class CSVReaderDelegate implements JavaDelegate { - - private final ImportProvider importProvider; - private final StorageService storageService; - - private final ProcessMapper mapper; - - protected CSVReaderDelegate(ImportProvider importProvider, StorageService storageService, ProcessMapper mapper) { - this.importProvider = importProvider; - this.storageService = storageService; - this.mapper = mapper; - } - - @Override - public void execute(DelegateExecution execution) throws IOException, CsvValidationException { - var batchImportSlug = (String) execution.getVariableLocal("batchImportSlug"); - var importConfigJson = getFromContext(execution); - - var batchImport = importProvider.lookup(batchImportSlug).get(); - log.debug("{}: Processing transaction import CSV {}", execution.getCurrentActivityName(), batchImport.getSlug()); - if (importConfigJson == null) { - throw new IllegalStateException("Cannot run account extraction without actual configuration."); - } - - var mappingIndices = computeIndices(importConfigJson.getColumnRoles()); - - beforeProcess(execution, importConfigJson); - - try (var inputStream = storageService.read(batchImport.getFileCode()) - .map(bytes -> new InputStreamReader(new ByteArrayInputStream(bytes))) - .get()) { - var reader = new CSVReaderBuilder(inputStream) - .withCSVParser(new CSVParserBuilder() - .withSeparator(importConfigJson.getDelimiter()) - .build()) - .build(); - - if (importConfigJson.isHeaders()) { - reader.skip(1); - } - - var line = reader.readNext(); - while (line != null) { - if (lineMatcherRequirements(line, mappingIndices.size())) { - var transaction = transform(line, mappingIndices, importConfigJson); - lineRead(execution, transaction); - } - line = reader.readNext(); - } - } - - afterProcess(execution); - } - - protected abstract void beforeProcess(DelegateExecution execution, ImportConfigJson configJson); - - protected abstract void lineRead(DelegateExecution execution, ParsedTransaction parsedTransaction); - - protected abstract void afterProcess(DelegateExecution execution); - - private boolean lineMatcherRequirements(String[] line, int mappingIndicesCount) { - if (line.length == 1) { - log.debug("Invalid short line in CSV {}", String.join(", ", line)); - } else if (line.length < mappingIndicesCount) { - log.debug("Line contains to few columns got {} expected {}, `{}`", - line.length, - mappingIndicesCount, - String.join(", ", line)); - } else { - return true; - } - - return false; - } - - private ImportConfigJson getFromContext(DelegateExecution execution) { - var rawEntity = execution.getVariable("importConfig"); - - return switch (rawEntity) { - case String json -> mapper.readSafe(json, ImportConfigJson.class); - case ImportConfigJson json -> json; - default -> throw new IllegalArgumentException("Unsupported import configuration provided."); - }; - } - - private ParsedTransaction transform( - String[] csvLine, - EnumMap mappings, - ImportConfigJson configJson) { - var amount = parseAmount(mappings, csvLine); - var date = parseDate( - mappings, - csvLine, - configJson.getDateFormat(), - ImportConfigJson.MappingRole.DATE); - var bookDate = parseDate( - mappings, - csvLine, - configJson.getDateFormat(), - ImportConfigJson.MappingRole.BOOK_DATE); - var interestDate = parseDate( - mappings, - csvLine, - configJson.getDateFormat(), - ImportConfigJson.MappingRole.INTEREST_DATE); - var opposingName = locateColumn(csvLine, mappings, ImportConfigJson.MappingRole.OPPOSING_NAME); - var opposingIBAN = locateColumn(csvLine, mappings, ImportConfigJson.MappingRole.OPPOSING_IBAN); - var description = locateColumn(csvLine, mappings, ImportConfigJson.MappingRole.DESCRIPTION); - - Transaction.Type type = amount >= 0 ? Transaction.Type.DEBIT : Transaction.Type.CREDIT; - if (mappings.containsKey(ImportConfigJson.MappingRole.CUSTOM_INDICATOR)) { - type = parseType(csvLine, mappings, configJson.getCustomIndicator()); - } - - return new ParsedTransaction( - amount, - type, - description, - date, - interestDate, - bookDate, - opposingIBAN, - opposingName); - } - - private EnumMap computeIndices( - Iterable columnRoles) { - EnumMap mapConfig = - new EnumMap<>(ImportConfigJson.MappingRole.class); - - var i = 0; - for (ImportConfigJson.MappingRole role : columnRoles) { - mapConfig.put(role, i); - i++; - } - - return mapConfig; - } - - private LocalDate parseDate( - EnumMap mappingIndices, String[] line, - String format, ImportConfigJson.MappingRole role) { - return Optional.ofNullable(locateColumn(line, mappingIndices, role)) - .map(str -> LocalDate.parse(str, DateTimeFormatter.ofPattern(format))) - .orElse(null); - } - - private Transaction.Type parseType( - String[] line, EnumMap mappingIndices, - ImportConfigJson.CustomIndicator customIndicator) { - return Optional.ofNullable(locateColumn(line, mappingIndices, ImportConfigJson.MappingRole.CUSTOM_INDICATOR)) - .map(str -> { - if (customIndicator.getCredit().equalsIgnoreCase(str)) { - return Transaction.Type.CREDIT; - } else { - return Transaction.Type.DEBIT; - } - }).orElseThrow(() -> new IllegalStateException("Incorrect custom indicator found")); - } - - private double parseAmount( - EnumMap mappingIndices, - String[] line) { - String amount = locateColumn(line, mappingIndices, ImportConfigJson.MappingRole.AMOUNT); - if (StringUtils.isEmpty(amount)) { - throw new IllegalStateException("Amount cannot be blank in import"); - } - - return Double.parseDouble(amount.replace(',', '.')); - } - - private String locateColumn( - String[] line, EnumMap mappingIndices, - ImportConfigJson.MappingRole column) { - if (mappingIndices.containsKey(column)) { - return line[mappingIndices.get(column)]; - } - - return null; - } -} diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java index 215afab6..16a6f3ef 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java @@ -1,46 +1,58 @@ package com.jongsoft.finance.bpmn.delegate.importer; -import com.jongsoft.finance.ProcessMapper; -import com.jongsoft.finance.StorageService; +import com.jongsoft.finance.importer.ImporterProvider; +import com.jongsoft.finance.importer.api.ImporterConfiguration; import com.jongsoft.finance.providers.ImportProvider; import com.jongsoft.finance.serialized.ExtractedAccountLookup; -import com.jongsoft.finance.serialized.ImportConfigJson; +import com.jongsoft.finance.serialized.ImportJobSettings; import jakarta.inject.Inject; import jakarta.inject.Singleton; import org.camunda.bpm.engine.delegate.DelegateExecution; +import org.camunda.bpm.engine.delegate.JavaDelegate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.HashSet; +import java.util.List; +import java.util.Set; @Singleton -public class ExtractAccountDetailsDelegate extends CSVReaderDelegate { +public class ExtractAccountDetailsDelegate implements JavaDelegate { + private final static Logger logger = LoggerFactory.getLogger(ExtractAccountDetailsDelegate.class); - @Inject - public ExtractAccountDetailsDelegate( - ImportProvider importProvider, - StorageService storageService, - ProcessMapper mapper) { - super(importProvider, storageService, mapper); - } + private final List> importerProviders; + private final ImportProvider importProvider; - @Override - protected void beforeProcess(DelegateExecution execution, ImportConfigJson configJson) { - execution.setVariableLocal("locatable", new HashSet<>()); - } - - @Override - protected void lineRead(DelegateExecution execution, ParsedTransaction parsedTransaction) { - var extraction = new ExtractedAccountLookup( - parsedTransaction.getOpposingName(), - parsedTransaction.getOpposingIBAN(), - parsedTransaction.getDescription()); - - @SuppressWarnings("unchecked") - var locatable = (HashSet) execution.getVariableLocal("locatable"); - locatable.add(extraction); + @Inject + public ExtractAccountDetailsDelegate(List> importerProviders, ImportProvider importProvider) { + this.importerProviders = importerProviders; + this.importProvider = importProvider; } @Override - protected void afterProcess(DelegateExecution execution) { - execution.setVariable("extractionResult", new HashSet<>()); + public void execute(DelegateExecution execution) throws Exception { + var batchImportSlug = (String) execution.getVariableLocal("batchImportSlug"); + var importJobSettings = (ImportJobSettings) execution.getVariable("importConfig"); + logger.debug("{}: Processing transaction import {}", execution.getCurrentActivityName(), batchImportSlug); + + var importJob = importProvider.lookup(batchImportSlug).get(); + Set locatable = new HashSet<>(); + importerProviders.stream() + .filter(provider -> provider.supports(importJobSettings.importConfiguration())) + .findFirst() + .ifPresentOrElse( + provider -> provider.readTransactions( + transactionDTO -> { + locatable.add(new ExtractedAccountLookup( + transactionDTO.opposingName(), + transactionDTO.opposingIBAN(), + transactionDTO.description())); + }, + importJobSettings.importConfiguration(), + importJob), + () -> logger.warn("No importer provider found for configuration: {}", importJobSettings.importConfiguration()) + ); + + execution.setVariableLocal("locatable", locatable); } } diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java index 0d676c7f..9128a463 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java @@ -1,33 +1,23 @@ package com.jongsoft.finance.bpmn.delegate.importer; -import com.jongsoft.finance.ProcessMapper; -import com.jongsoft.finance.StorageService; +import com.jongsoft.finance.importer.ImporterProvider; import com.jongsoft.finance.providers.ImportProvider; -import com.jongsoft.finance.serialized.ImportConfigJson; -import jakarta.inject.Inject; +import com.jongsoft.finance.serialized.ImportJobSettings; import jakarta.inject.Singleton; import lombok.extern.slf4j.Slf4j; import org.camunda.bpm.engine.delegate.DelegateExecution; import org.camunda.bpm.engine.delegate.JavaDelegate; -import java.nio.charset.StandardCharsets; - @Slf4j @Singleton public class LoadImporterConfiguration implements JavaDelegate { private final ImportProvider importProvider; - private final StorageService storageService; - private final ProcessMapper mapper; - - @Inject - public LoadImporterConfiguration( - ImportProvider importProvider, - StorageService storageService, - ProcessMapper mapper) { + private final ImporterProvider importerProvider; + + public LoadImporterConfiguration(ImportProvider importProvider, ImporterProvider importerProvider) { this.importProvider = importProvider; - this.storageService = storageService; - this.mapper = mapper; + this.importerProvider = importerProvider; } @Override @@ -41,15 +31,10 @@ public void execute(DelegateExecution delegateExecution) throws Exception { var importJob = importProvider.lookup(batchImportSlug) .getOrThrow(() -> new IllegalStateException("Cannot find batch import with slug " + batchImportSlug)); - var importConfig = readConfiguration(importJob.getConfig().getFileCode()); - - delegateExecution.setVariableLocal("importConfig", importConfig); - } + var configuration = importerProvider.loadConfiguration(importJob.getConfig()); - private ImportConfigJson readConfiguration(String fileCode) { - return storageService.read(fileCode) - .map(bytes -> new String(bytes, StandardCharsets.UTF_8)) - .map(json -> mapper.readSafe(json, ImportConfigJson.class)) - .getOrThrow(() -> new IllegalStateException("Cannot read import configuration from file " + fileCode)); + delegateExecution.setVariableLocal( + "importConfig", + new ImportJobSettings(configuration, false, false, null)); } } diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ReadTransactionLogDelegate.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ReadTransactionLogDelegate.java index b40fa31e..048c9ad2 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ReadTransactionLogDelegate.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ReadTransactionLogDelegate.java @@ -2,8 +2,10 @@ import com.jongsoft.finance.ProcessMapper; import com.jongsoft.finance.StorageService; +import com.jongsoft.finance.importer.ImporterProvider; +import com.jongsoft.finance.importer.api.ImporterConfiguration; import com.jongsoft.finance.providers.ImportProvider; -import com.jongsoft.finance.serialized.ImportConfigJson; +import com.jongsoft.finance.serialized.ImportJobSettings; import jakarta.inject.Inject; import jakarta.inject.Singleton; import lombok.extern.slf4j.Slf4j; @@ -12,46 +14,56 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.List; @Slf4j @Singleton -public class ReadTransactionLogDelegate extends CSVReaderDelegate implements JavaDelegate { +public class ReadTransactionLogDelegate implements JavaDelegate { + private final List> importerProviders; + private final ImportProvider importProvider; private final StorageService storageService; private final ProcessMapper mapper; @Inject - public ReadTransactionLogDelegate(ImportProvider importProvider, StorageService storageService, ProcessMapper mapper) { - super(importProvider, storageService, mapper); + public ReadTransactionLogDelegate( + List> importerProviders, + ImportProvider importProvider, + StorageService storageService, + ProcessMapper mapper) { + this.importerProviders = importerProviders; + this.importProvider = importProvider; this.storageService = storageService; this.mapper = mapper; } @Override - protected void beforeProcess(DelegateExecution execution, ImportConfigJson configJson) { - log.debug("Setting up reader for import file with configuration {}", configJson); - execution.setVariableLocal("generateAccounts", configJson.isGenerateAccounts()); - execution.setVariableLocal("applyRules", configJson.isApplyRules()); - execution.setVariableLocal("targetAccountId", configJson.getAccountId()); - execution.setVariableLocal("storageTokens", new ArrayList()); - } - - @Override - protected void lineRead(DelegateExecution execution, ParsedTransaction parsedTransaction) { - log.debug("Read line {} of file import", parsedTransaction); + public void execute(DelegateExecution execution) throws Exception { + var batchImportSlug = (String) execution.getVariableLocal("batchImportSlug"); + var importJobSettings = (ImportJobSettings) execution.getVariable("importConfig"); + log.debug("{}: Processing transaction import {}", execution.getCurrentActivityName(), batchImportSlug); - var serialized = mapper.writeSafe(parsedTransaction) - .getBytes(StandardCharsets.UTF_8); + var importJob = importProvider.lookup(batchImportSlug).get(); + List storageTokens = new ArrayList<>(); - String storageToken = storageService.store(serialized); + importerProviders.stream() + .filter(provider -> provider.supports(importJobSettings.importConfiguration())) + .findFirst() + .ifPresentOrElse( + provider -> provider.readTransactions( + transactionDTO -> { + var serialized = mapper.writeSafe(transactionDTO) + .getBytes(StandardCharsets.UTF_8); + storageTokens.add(storageService.store(serialized)); + }, + importJobSettings.importConfiguration(), + importJob), + () -> log.warn("No importer provider found for configuration: {}", importJobSettings.importConfiguration()) + ); - var storageTokens = (ArrayList) execution.getVariableLocal("storageTokens"); - storageTokens.add(storageToken); + execution.setVariableLocal("generateAccounts", importJobSettings.generateAccounts()); + execution.setVariableLocal("applyRules", importJobSettings.applyRules()); + execution.setVariableLocal("targetAccountId", importJobSettings.accountId()); + execution.setVariableLocal("storageTokens", storageTokens); } - - @Override - protected void afterProcess(DelegateExecution execution) { - // no specific implementation required - } - } diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportConfigJson.java b/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportConfigJson.java deleted file mode 100644 index 699f054b..00000000 --- a/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportConfigJson.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.jongsoft.finance.serialized; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.jongsoft.finance.ProcessVariable; -import io.micronaut.serde.annotation.Serdeable; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -import java.io.Serializable; -import java.util.List; - -@Getter -@Setter -@Serdeable -@ToString(of = {"accountId", "headers", "dateFormat", "delimiter"}) -public class ImportConfigJson implements ProcessVariable { - - public enum MappingRole { - IGNORE("_ignore"), - DATE("transaction-date"), - BOOK_DATE("booking-date"), - INTEREST_DATE("interest-date"), - OPPOSING_NAME("opposing-name"), - OPPOSING_IBAN("opposing-iban"), - ACCOUNT_IBAN("account-iban"), - AMOUNT("amount"), - CUSTOM_INDICATOR("custom-indicator"), - DESCRIPTION("description"); - - private final String label; - - MappingRole(String label) { - this.label = label; - } - - public String getLabel() { - return label; - } - - public static MappingRole value(String source) { - for (MappingRole role : values()) { - if (role.label.equalsIgnoreCase(source)) { - return role; - } - } - throw new IllegalStateException("No mapping role found for " + source); - } - } - - @JsonProperty("has-headers") - private boolean headers; - - @JsonProperty("apply-rules") - private boolean applyRules; - - @JsonProperty("generate-accounts") - private boolean generateAccounts; - - @JsonProperty("date-format") - private String dateFormat; - - @JsonProperty("delimiter") - private Character delimiter; - - @JsonProperty("column-roles") - private List columnRoles; - - @JsonProperty("custom-indicator") - private CustomIndicator customIndicator; - - private Long accountId; - - @Getter - @Setter - @Serdeable - public static class CustomIndicator implements Serializable { - @JsonProperty("deposit") - private String deposit; - - @JsonProperty("credit") - private String credit; - } - -} diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportJobSettings.java b/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportJobSettings.java new file mode 100644 index 00000000..41704462 --- /dev/null +++ b/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportJobSettings.java @@ -0,0 +1,16 @@ +package com.jongsoft.finance.serialized; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.jongsoft.finance.importer.api.ImporterConfiguration; +import io.micronaut.serde.annotation.Serdeable; + +@Serdeable +public record ImportJobSettings( + @JsonProperty + ImporterConfiguration importConfiguration, + @JsonProperty + boolean applyRules, + @JsonProperty + boolean generateAccounts, + @JsonProperty + Long accountId) {} diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/serialized/serde/MappingRoleSerde.java b/bpmn-process/src/main/java/com/jongsoft/finance/serialized/serde/MappingRoleSerde.java deleted file mode 100644 index fe62aec7..00000000 --- a/bpmn-process/src/main/java/com/jongsoft/finance/serialized/serde/MappingRoleSerde.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jongsoft.finance.serialized.serde; - -import com.jongsoft.finance.serialized.ImportConfigJson; -import io.micronaut.core.annotation.NonNull; -import io.micronaut.core.annotation.Nullable; -import io.micronaut.core.type.Argument; -import io.micronaut.serde.Decoder; -import io.micronaut.serde.Encoder; -import io.micronaut.serde.Serde; -import jakarta.inject.Singleton; - -import java.io.IOException; - -@Singleton -public class MappingRoleSerde implements Serde { - @Override - public @Nullable ImportConfigJson.MappingRole deserialize( - @NonNull Decoder decoder, - @NonNull DecoderContext context, - @NonNull Argument type) throws IOException { - return ImportConfigJson.MappingRole.value(decoder.decodeString()); - } - - @Override - public void serialize( - @NonNull Encoder encoder, - @NonNull EncoderContext context, - @NonNull Argument type, ImportConfigJson.@NonNull MappingRole value) throws IOException { - encoder.encodeString(value.getLabel()); - } -} diff --git a/bpmn-process/src/main/resources/bpmn/transaction/import-job.bpmn b/bpmn-process/src/main/resources/bpmn/transaction/import-job.bpmn index af4f6ede..d245aa17 100644 --- a/bpmn-process/src/main/resources/bpmn/transaction/import-job.bpmn +++ b/bpmn-process/src/main/resources/bpmn/transaction/import-job.bpmn @@ -38,7 +38,7 @@ When calling the flow the following variables must be set: - ${importConfiguration.getAccountId()} + ${importConfiguration.accountId()} ${id} @@ -60,9 +60,6 @@ When calling the flow the following variables must be set: ${importJobSlug} ${importConfiguration} - storageTokens - generateAccounts - applyRules ${storageTokens} ${generateAccounts} ${applyRules} @@ -131,7 +128,6 @@ When calling the flow the following variables must be set: ${account_extract.name} ${account_extract.description} - accountId ${accountId} @@ -169,8 +165,6 @@ When calling the flow the following variables must be set: ${importJobSlug} ${importConfiguration} - extractionResult - ${extractionResult} ${locatable} @@ -247,7 +241,6 @@ When calling the flow the following variables must be set: ${transactionToken} - transaction ${transaction} diff --git a/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ImportJobIT.java b/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ImportJobIT.java index 550b258f..b2d495e8 100644 --- a/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ImportJobIT.java +++ b/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ImportJobIT.java @@ -8,7 +8,7 @@ import com.jongsoft.finance.domain.importer.BatchImportConfig; import com.jongsoft.finance.domain.transaction.Transaction; import com.jongsoft.finance.providers.AccountProvider; -import com.jongsoft.finance.serialized.ImportConfigJson; +import com.jongsoft.finance.serialized.ImportJobSettings; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.DisplayName; @@ -42,10 +42,14 @@ void runImportAccountNotFound(RuntimeContext context) { process .task("task_configure") - .updateVariable("initialConfig", "updatedConfig", config -> { - config.setAccountId(TARGET_ACCOUNT_ID); - return config; - }) + .updateVariable( + "initialConfig", + "updatedConfig", + config -> new ImportJobSettings( + config.importConfiguration(), + config.applyRules(), + config.generateAccounts(), + TARGET_ACCOUNT_ID)) .complete(); // Task should be active again @@ -70,10 +74,14 @@ void runWithAccountMappingAdjustments(RuntimeContext context) { )); process.task("task_configure") - .updateVariable("initialConfig", "updatedConfig", config -> { - config.setAccountId(TARGET_ACCOUNT_ID); - return config; - }) + .updateVariable( + "initialConfig", + "updatedConfig", + config -> new ImportJobSettings( + config.importConfiguration(), + config.applyRules(), + config.generateAccounts(), + TARGET_ACCOUNT_ID)) .complete(); process.task("confirm_mappings") @@ -118,11 +126,14 @@ void runWithManualAccountCreate(RuntimeContext context) { )); process.task("task_configure") - .updateVariable("initialConfig", "updatedConfig", config -> { - config.setAccountId(TARGET_ACCOUNT_ID); - config.setGenerateAccounts(false); - return config; - }) + .updateVariable( + "initialConfig", + "updatedConfig", + config -> new ImportJobSettings( + config.importConfiguration(), + config.applyRules(), + false, + TARGET_ACCOUNT_ID)) .complete(); process.task("confirm_mappings") @@ -165,11 +176,14 @@ void runWithAutomatedAccountCreation(RuntimeContext context) { )); process.task("task_configure") - .updateVariable("initialConfig", "updatedConfig", config -> { - config.setAccountId(TARGET_ACCOUNT_ID); - config.setGenerateAccounts(true); - return config; - }) + .updateVariable( + "initialConfig", + "updatedConfig", + config -> new ImportJobSettings( + config.importConfiguration(), + config.applyRules(), + true, + TARGET_ACCOUNT_ID)) .complete(); process.task("confirm_mappings") diff --git a/bpmn-process/src/test/resources/import-test/import-test.csv b/bpmn-process/src/test/resources/import-test/import-test.csv index 5a31c134..bcb4a999 100644 --- a/bpmn-process/src/test/resources/import-test/import-test.csv +++ b/bpmn-process/src/test/resources/import-test/import-test.csv @@ -1,5 +1,5 @@ -Datum,Naam/Omschrijving,Rekening,Tegenrekening,Code,Af/Bij,Bedrag,Valuta,Mutatiesoort,Betalingskenmerk,Mededeling -20160531,Janssen PA,NL20INGB0001234567,NL69INGB0123454789,GT,Bij,"14,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399,, -20160531,MW GA Pieterse,NL20INGB0001234567,NL71INGB0009876543,GT,Af,"283,90",Internetbankieren,Naam: Mw G A Pieterse Omschrijving: inzake bestelling IBAN: NL71INGB0009876543,, -20160531,P. Post,NL20INGB0001234567,NL69INGB0123456789,GT,Bij,"14,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399,, -20160525,Janssen PA,NL20INGB0001234567,NL69INGB0123454789,GT,Bij,"12,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399,, +Datum,Naam/Omschrijving,Rekening,Tegenrekening,Code,Af/Bij,Bedrag,Valuta,Mutatiesoort +20160531,Janssen PA,NL20INGB0001234567,NL69INGB0123454789,GT,Bij,"14,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399 +20160531,MW GA Pieterse,NL20INGB0001234567,NL71INGB0009876543,GT,Af,"283,90",Internetbankieren,Naam: Mw G A Pieterse Omschrijving: inzake bestelling IBAN: NL71INGB0009876543 +20160531,P. Post,NL20INGB0001234567,NL69INGB0123456789,GT,Bij,"14,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399 +20160525,Janssen PA,NL20INGB0001234567,NL69INGB0123454789,GT,Bij,"12,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399 diff --git a/fintrack-api/build.gradle.kts b/fintrack-api/build.gradle.kts index e9593049..da7be999 100644 --- a/fintrack-api/build.gradle.kts +++ b/fintrack-api/build.gradle.kts @@ -44,6 +44,8 @@ dependencies { implementation(project(":jpa-repository")) implementation(project(":rule-engine")) implementation(project(":bpmn-process")) + implementation(project(":transaction-importer:transaction-importer-api")) + implementation(project(":transaction-importer:transaction-importer-csv")) // needed for application.yml runtimeOnly(mn.snakeyaml) diff --git a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImportProvider.java b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImporterProvider.java similarity index 57% rename from transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImportProvider.java rename to transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImporterProvider.java index 1ae9cce5..add2f5c8 100644 --- a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImportProvider.java +++ b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImporterProvider.java @@ -5,10 +5,12 @@ import com.jongsoft.finance.importer.api.ImporterConfiguration; import com.jongsoft.finance.importer.api.TransactionConsumer; -public interface ImportProvider { +public interface ImporterProvider { - void readTransactions(TransactionConsumer consumer, T updatedConfiguration, BatchImport importJob); + void readTransactions(TransactionConsumer consumer, ImporterConfiguration updatedConfiguration, BatchImport importJob); T loadConfiguration(BatchImportConfig batchImportConfig); + boolean supports(X configuration); + } diff --git a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/ImporterConfiguration.java b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/ImporterConfiguration.java index badd7d99..d4dcf160 100644 --- a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/ImporterConfiguration.java +++ b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/ImporterConfiguration.java @@ -1,4 +1,7 @@ package com.jongsoft.finance.importer.api; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "type") public interface ImporterConfiguration { } diff --git a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVImportProvider.java b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVImportProvider.java index f750735c..5edae6f7 100644 --- a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVImportProvider.java +++ b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVImportProvider.java @@ -4,7 +4,8 @@ import com.jongsoft.finance.core.TransactionType; import com.jongsoft.finance.domain.importer.BatchImport; import com.jongsoft.finance.domain.importer.BatchImportConfig; -import com.jongsoft.finance.importer.ImportProvider; +import com.jongsoft.finance.importer.ImporterProvider; +import com.jongsoft.finance.importer.api.ImporterConfiguration; import com.jongsoft.finance.importer.api.TransactionConsumer; import com.jongsoft.finance.importer.api.TransactionDTO; import com.jongsoft.lang.Control; @@ -25,7 +26,7 @@ import java.util.function.Function; @Singleton -public class CSVImportProvider implements ImportProvider { +public class CSVImportProvider implements ImporterProvider { private final Logger logger = LoggerFactory.getLogger(CSVImportProvider.class); private final StorageService storageService; @@ -38,8 +39,9 @@ public CSVImportProvider(StorageService storageService, ObjectMapper objectMappe } @Override - public void readTransactions(TransactionConsumer consumer, CSVConfiguration configuration, BatchImport importJob) { + public void readTransactions(TransactionConsumer consumer, ImporterConfiguration configuration, BatchImport importJob) { logger.info("Reading transactions from CSV file: {}", importJob.getSlug()); + var csvConfiguration = (CSVConfiguration) configuration; try { var inputStream = storageService.read(importJob.getFileCode()) @@ -49,23 +51,26 @@ public void readTransactions(TransactionConsumer consumer, CSVConfiguration conf try (var reader = new CSVReaderBuilder(inputStream) .withCSVParser(new CSVParserBuilder() - .withSeparator(configuration.delimiter()) + .withSeparator(csvConfiguration.delimiter()) .build()) .build()) { - if (configuration.headers()) { + if (csvConfiguration.headers()) { logger.debug("CSV file has headers, skipping first line"); reader.skip(1); } String[] line; while ((line = reader.readNext()) != null) { - if (line.length != configuration.columnRoles().size()) { - logger.warn("Skipping line, columns found {} but expected is {}: {}", line.length, configuration.columnRoles().size(), line); + if (line.length != csvConfiguration.columnRoles().size()) { + logger.warn("Skipping line, columns found {} but expected is {}: {}", + line.length, + csvConfiguration.columnRoles().size(), + line); continue; } - consumer.accept(readLine(line, configuration)); + consumer.accept(readLine(line, csvConfiguration)); } } } catch (IOException | CsvValidationException e) { @@ -91,6 +96,11 @@ public CSVConfiguration loadConfiguration(BatchImportConfig batchImportConfig) { } } + @Override + public boolean supports(X configuration) { + return configuration instanceof CSVConfiguration; + } + private TransactionDTO readLine(String[] line, CSVConfiguration configuration) { Function columnLocator = (role) -> Control.Try(() -> line[configuration.columnRoles().indexOf(role)]).recover(x -> null).get(); Function parseDate = (date) -> date != null ? LocalDate.parse(date, DateTimeFormatter.ofPattern(configuration.dateFormat())) : null; From 5289def70bfc0adb8e5b6b68ac3c51b8c9aa2896 Mon Sep 17 00:00:00 2001 From: Gerben Jongerius Date: Thu, 4 Apr 2024 14:31:54 +0200 Subject: [PATCH 4/8] Enforce that the ImportConfig contains the type of import that should be run. --- .../importer/LoadImporterConfiguration.java | 28 ++++++++++++++----- .../finance/bpmn/ApplicationContext.java | 4 +-- .../jongsoft/finance/bpmn/ImportJobIT.java | 1 + .../domain/importer/BatchImportConfig.java | 10 ++++--- .../finance/domain/user/UserAccount.java | 4 +-- .../importer/CreateConfigurationCommand.java | 2 +- ....java => ImportConfigurationProvider.java} | 2 +- .../finance/domain/user/UserAccountTest.java | 4 +-- .../rest/importer/BatchImportResource.java | 7 +++-- .../CSVImporterConfigCreateRequest.java | 2 ++ .../importer/BatchImportResourceTest.java | 14 ++++++---- .../importer/CreateConfigurationHandler.java | 5 ++-- .../jpa/importer/CreateImportJobHandler.java | 6 ++-- ...va => ImportConfigurationProviderJpa.java} | 20 ++++++------- ...CSVImportConfig.java => ImportConfig.java} | 10 +++++-- .../jpa/importer/entity/ImportJpa.java | 4 +-- .../V20240404131810__add_import_type.sql | 5 ++++ .../importer/CSVConfigEventListenerIT.java | 8 +++--- .../jpa/importer/CSVConfigProviderJpaIT.java | 4 +-- .../finance/importer/ImporterProvider.java | 4 +++ .../importer/csv/CSVImportProviderTest.java | 6 ++++ 21 files changed, 96 insertions(+), 54 deletions(-) rename domain/src/main/java/com/jongsoft/finance/providers/{CSVConfigProvider.java => ImportConfigurationProvider.java} (86%) rename jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/{CSVConfigProviderJpa.java => ImportConfigurationProviderJpa.java} (74%) rename jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/{CSVImportConfig.java => ImportConfig.java} (74%) create mode 100644 jpa-repository/src/main/resources/db/migration/V20240404131810__add_import_type.sql diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java index 9128a463..c27616ec 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java @@ -8,14 +8,16 @@ import org.camunda.bpm.engine.delegate.DelegateExecution; import org.camunda.bpm.engine.delegate.JavaDelegate; +import java.util.List; + @Slf4j @Singleton public class LoadImporterConfiguration implements JavaDelegate { private final ImportProvider importProvider; - private final ImporterProvider importerProvider; + private final List> importerProvider; - public LoadImporterConfiguration(ImportProvider importProvider, ImporterProvider importerProvider) { + public LoadImporterConfiguration(ImportProvider importProvider, List> importerProvider) { this.importProvider = importProvider; this.importerProvider = importerProvider; } @@ -31,10 +33,22 @@ public void execute(DelegateExecution delegateExecution) throws Exception { var importJob = importProvider.lookup(batchImportSlug) .getOrThrow(() -> new IllegalStateException("Cannot find batch import with slug " + batchImportSlug)); - var configuration = importerProvider.loadConfiguration(importJob.getConfig()); - - delegateExecution.setVariableLocal( - "importConfig", - new ImportJobSettings(configuration, false, false, null)); + importerProvider.stream() + .filter(importer -> importer.getImporterType().equalsIgnoreCase(importJob.getConfig().getType())) + .findFirst() + .ifPresentOrElse( + importer -> { + delegateExecution.setVariableLocal( + "importConfig", + new ImportJobSettings( + importer.loadConfiguration(importJob.getConfig()), + false, + false, + null)); + }, + () -> { + throw new IllegalStateException("Cannot find importer for type " + importJob.getConfig().getType()); + } + ); } } diff --git a/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ApplicationContext.java b/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ApplicationContext.java index 0c80eb1c..1fc44e5d 100644 --- a/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ApplicationContext.java +++ b/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ApplicationContext.java @@ -142,8 +142,8 @@ ImportProvider importProvider() { } @Singleton - CSVConfigProvider csvConfigProvider() { - return Mockito.mock(CSVConfigProvider.class); + ImportConfigurationProvider csvConfigProvider() { + return Mockito.mock(ImportConfigurationProvider.class); } @Singleton diff --git a/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ImportJobIT.java b/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ImportJobIT.java index b2d495e8..209b5857 100644 --- a/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ImportJobIT.java +++ b/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ImportJobIT.java @@ -283,6 +283,7 @@ private BatchImport createBatchImport() { .created(new Date()) .fileCode(CSV_FILE_CODE) .config(BatchImportConfig.builder() + .type("CSVImportProvider") .fileCode(JSON_FILE_CODE) .build()) .build(); diff --git a/domain/src/main/java/com/jongsoft/finance/domain/importer/BatchImportConfig.java b/domain/src/main/java/com/jongsoft/finance/domain/importer/BatchImportConfig.java index 04335821..dc520be5 100644 --- a/domain/src/main/java/com/jongsoft/finance/domain/importer/BatchImportConfig.java +++ b/domain/src/main/java/com/jongsoft/finance/domain/importer/BatchImportConfig.java @@ -22,21 +22,23 @@ public class BatchImportConfig implements AggregateBase, Serializable { private String name; private String fileCode; + private String type; private transient UserAccount user; @BusinessMethod - public BatchImportConfig(UserAccount user, String name, String fileCode) { + public BatchImportConfig(UserAccount user, String type, String name, String fileCode) { this.user = user; this.name = name; this.fileCode = fileCode; + this.type = type; EventBus.getBus().send( - new CreateConfigurationCommand(name, fileCode)); + new CreateConfigurationCommand(type, name, fileCode)); } - public BatchImport createImport(String content) { - return new BatchImport(this, this.user, content); + public BatchImport createImport(String fileCode) { + return new BatchImport(this, this.user, fileCode); } } diff --git a/domain/src/main/java/com/jongsoft/finance/domain/user/UserAccount.java b/domain/src/main/java/com/jongsoft/finance/domain/user/UserAccount.java index e5294494..16142a5e 100644 --- a/domain/src/main/java/com/jongsoft/finance/domain/user/UserAccount.java +++ b/domain/src/main/java/com/jongsoft/finance/domain/user/UserAccount.java @@ -182,12 +182,12 @@ public TransactionRule createRule(String name, boolean restrictive) { * @return the newly created configuration */ @BusinessMethod - public BatchImportConfig createImportConfiguration(String name, String fileCode) { + public BatchImportConfig createImportConfiguration(String type, String name, String fileCode) { if (notFullUser()) { throw StatusException.notAuthorized("User cannot create import configuration, incorrect privileges."); } - return new BatchImportConfig(this, name, fileCode); + return new BatchImportConfig(this, type, name, fileCode); } /** diff --git a/domain/src/main/java/com/jongsoft/finance/messaging/commands/importer/CreateConfigurationCommand.java b/domain/src/main/java/com/jongsoft/finance/messaging/commands/importer/CreateConfigurationCommand.java index f4450a70..a1787d6c 100644 --- a/domain/src/main/java/com/jongsoft/finance/messaging/commands/importer/CreateConfigurationCommand.java +++ b/domain/src/main/java/com/jongsoft/finance/messaging/commands/importer/CreateConfigurationCommand.java @@ -2,5 +2,5 @@ import com.jongsoft.finance.core.ApplicationEvent; -public record CreateConfigurationCommand(String name, String fileCode) implements ApplicationEvent { +public record CreateConfigurationCommand(String type, String name, String fileCode) implements ApplicationEvent { } diff --git a/domain/src/main/java/com/jongsoft/finance/providers/CSVConfigProvider.java b/domain/src/main/java/com/jongsoft/finance/providers/ImportConfigurationProvider.java similarity index 86% rename from domain/src/main/java/com/jongsoft/finance/providers/CSVConfigProvider.java rename to domain/src/main/java/com/jongsoft/finance/providers/ImportConfigurationProvider.java index f74e51ba..2c8397f3 100644 --- a/domain/src/main/java/com/jongsoft/finance/providers/CSVConfigProvider.java +++ b/domain/src/main/java/com/jongsoft/finance/providers/ImportConfigurationProvider.java @@ -4,7 +4,7 @@ import com.jongsoft.lang.collection.Sequence; import com.jongsoft.lang.control.Optional; -public interface CSVConfigProvider { +public interface ImportConfigurationProvider { Optional lookup(String name); Sequence lookup(); diff --git a/domain/src/test/java/com/jongsoft/finance/domain/user/UserAccountTest.java b/domain/src/test/java/com/jongsoft/finance/domain/user/UserAccountTest.java index 0c1db5d1..4917054b 100644 --- a/domain/src/test/java/com/jongsoft/finance/domain/user/UserAccountTest.java +++ b/domain/src/test/java/com/jongsoft/finance/domain/user/UserAccountTest.java @@ -118,7 +118,7 @@ void createCategory_NotAllowed() { @Test void createBatchConfiguration() { - final BatchImportConfig configuration = fullAccount.createImportConfiguration("test-config", "file-code-sample"); + final BatchImportConfig configuration = fullAccount.createImportConfiguration("CSV", "test-config", "file-code-sample"); assertThat(configuration.getName()).isEqualTo("test-config"); assertThat(configuration.getFileCode()).isEqualTo("file-code-sample"); @@ -128,7 +128,7 @@ void createBatchConfiguration() { @Test void createBatchConfiguration_NotAllowed() { var thrown = assertThrows(StatusException.class, - () -> readOnlyAccount.createImportConfiguration("test-config", "file-code-sample")); + () -> readOnlyAccount.createImportConfiguration("CSV", "test-config", "file-code-sample")); assertThat(thrown.getMessage()).isEqualTo("User cannot create import configuration, incorrect privileges."); } diff --git a/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/BatchImportResource.java b/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/BatchImportResource.java index 23cffdb2..d3534296 100644 --- a/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/BatchImportResource.java +++ b/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/BatchImportResource.java @@ -1,7 +1,7 @@ package com.jongsoft.finance.rest.importer; import com.jongsoft.finance.core.exception.StatusException; -import com.jongsoft.finance.providers.CSVConfigProvider; +import com.jongsoft.finance.providers.ImportConfigurationProvider; import com.jongsoft.finance.providers.ImportProvider; import com.jongsoft.finance.providers.SettingProvider; import com.jongsoft.finance.rest.model.CSVImporterConfigResponse; @@ -26,13 +26,13 @@ public class BatchImportResource { private final CurrentUserProvider currentUserProvider; - private final CSVConfigProvider csvConfigProvider; + private final ImportConfigurationProvider csvConfigProvider; private final ImportProvider importProvider; private final SettingProvider settingProvider; public BatchImportResource( CurrentUserProvider currentUserProvider, - CSVConfigProvider csvConfigProvider, + ImportConfigurationProvider csvConfigProvider, ImportProvider importProvider, SettingProvider settingProvider) { this.currentUserProvider = currentUserProvider; @@ -125,6 +125,7 @@ CSVImporterConfigResponse createConfig(@Valid @Body CSVImporterConfigCreateReque } return new CSVImporterConfigResponse(currentUserProvider.currentUser().createImportConfiguration( + request.type(), request.name(), request.fileCode())); } diff --git a/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/CSVImporterConfigCreateRequest.java b/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/CSVImporterConfigCreateRequest.java index c8469c6f..b2b2a871 100644 --- a/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/CSVImporterConfigCreateRequest.java +++ b/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/CSVImporterConfigCreateRequest.java @@ -5,6 +5,8 @@ @Serdeable.Deserializable record CSVImporterConfigCreateRequest( + @NotBlank + String type, @NotBlank String name, @NotBlank diff --git a/fintrack-api/src/test/java/com/jongsoft/finance/rest/importer/BatchImportResourceTest.java b/fintrack-api/src/test/java/com/jongsoft/finance/rest/importer/BatchImportResourceTest.java index 98d8ff79..6ffee29a 100644 --- a/fintrack-api/src/test/java/com/jongsoft/finance/rest/importer/BatchImportResourceTest.java +++ b/fintrack-api/src/test/java/com/jongsoft/finance/rest/importer/BatchImportResourceTest.java @@ -4,14 +4,13 @@ import com.jongsoft.finance.core.DateUtils; import com.jongsoft.finance.domain.importer.BatchImport; import com.jongsoft.finance.domain.importer.BatchImportConfig; -import com.jongsoft.finance.providers.CSVConfigProvider; +import com.jongsoft.finance.providers.ImportConfigurationProvider; import com.jongsoft.finance.providers.ImportProvider; import com.jongsoft.finance.rest.TestSetup; import com.jongsoft.lang.Collections; import com.jongsoft.lang.Control; import io.micronaut.context.annotation.Replaces; import io.micronaut.test.annotation.MockBean; -import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.restassured.specification.RequestSpecification; import jakarta.inject.Inject; import org.hamcrest.Matchers; @@ -29,12 +28,12 @@ class BatchImportResourceTest extends TestSetup { @Inject private ImportProvider importProvider; @Inject - private CSVConfigProvider csvConfigProvider; + private ImportConfigurationProvider csvConfigProvider; @Replaces @MockBean - CSVConfigProvider configProvider() { - return Mockito.mock(CSVConfigProvider.class); + ImportConfigurationProvider configProvider() { + return Mockito.mock(ImportConfigurationProvider.class); } @Replaces @@ -209,7 +208,10 @@ void createConfig(RequestSpecification spec) { // @formatter:off spec .given() - .body(Map.of("name", "sample-configuration", "fileCode", "token-sample")) + .body(Map.of( + "type", "csv", + "name", "sample-configuration", + "fileCode", "token-sample")) .when() .put("/api/import/config") .then() diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateConfigurationHandler.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateConfigurationHandler.java index 82090e84..bf6257b0 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateConfigurationHandler.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateConfigurationHandler.java @@ -1,7 +1,7 @@ package com.jongsoft.finance.jpa.importer; import com.jongsoft.finance.annotation.BusinessEventListener; -import com.jongsoft.finance.jpa.importer.entity.CSVImportConfig; +import com.jongsoft.finance.jpa.importer.entity.ImportConfig; import com.jongsoft.finance.jpa.reactive.ReactiveEntityManager; import com.jongsoft.finance.jpa.user.entity.UserAccountJpa; import com.jongsoft.finance.messaging.CommandHandler; @@ -32,9 +32,10 @@ public CreateConfigurationHandler(ReactiveEntityManager entityManager, Authentic public void handle(CreateConfigurationCommand command) { log.info("[{}] - Processing CSV configuration create event", command.name()); - var entity = CSVImportConfig.builder() + var entity = ImportConfig.builder() .fileCode(command.fileCode()) .name(command.name()) + .type(command.type()) .user(entityManager.get(UserAccountJpa.class, Collections.Map("username", authenticationFacade.authenticated()))) .build(); diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateImportJobHandler.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateImportJobHandler.java index 42ae7cf6..df8f781f 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateImportJobHandler.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateImportJobHandler.java @@ -2,7 +2,7 @@ import com.jongsoft.finance.annotation.BusinessEventListener; import com.jongsoft.finance.core.exception.StatusException; -import com.jongsoft.finance.jpa.importer.entity.CSVImportConfig; +import com.jongsoft.finance.jpa.importer.entity.ImportConfig; import com.jongsoft.finance.jpa.importer.entity.ImportJpa; import com.jongsoft.finance.jpa.reactive.ReactiveEntityManager; import com.jongsoft.finance.messaging.CommandHandler; @@ -29,8 +29,8 @@ public CreateImportJobHandler(ReactiveEntityManager entityManager) { public void handle(CreateImportJobCommand command) { log.info("[{}] - Processing import create event", command.slug()); - var configJpa = entityManager.blocking() - .hql("from CSVImportConfig where id = :id") + var configJpa = entityManager.blocking() + .hql("from ImportConfig where id = :id") .set("id", command.configId()) .maybe() .getOrThrow(() -> diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpa.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportConfigurationProviderJpa.java similarity index 74% rename from jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpa.java rename to jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportConfigurationProviderJpa.java index 90421ffa..b6042e5c 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpa.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportConfigurationProviderJpa.java @@ -2,9 +2,9 @@ import com.jongsoft.finance.domain.importer.BatchImportConfig; import com.jongsoft.finance.domain.user.UserAccount; -import com.jongsoft.finance.jpa.importer.entity.CSVImportConfig; +import com.jongsoft.finance.jpa.importer.entity.ImportConfig; import com.jongsoft.finance.jpa.reactive.ReactiveEntityManager; -import com.jongsoft.finance.providers.CSVConfigProvider; +import com.jongsoft.finance.providers.ImportConfigurationProvider; import com.jongsoft.finance.security.AuthenticationFacade; import com.jongsoft.lang.collection.Sequence; import com.jongsoft.lang.control.Optional; @@ -15,27 +15,27 @@ @Slf4j @ReadOnly @Singleton -public class CSVConfigProviderJpa implements CSVConfigProvider { +public class ImportConfigurationProviderJpa implements ImportConfigurationProvider { private final ReactiveEntityManager entityManager; private final AuthenticationFacade authenticationFacade; - public CSVConfigProviderJpa(ReactiveEntityManager entityManager, AuthenticationFacade authenticationFacade) { + public ImportConfigurationProviderJpa(ReactiveEntityManager entityManager, AuthenticationFacade authenticationFacade) { this.entityManager = entityManager; this.authenticationFacade = authenticationFacade; } @Override public Optional lookup(String name) { - log.trace("CSVConfiguration lookup by name {}", name); + log.trace("Import configuration lookup by name {}", name); var hql = """ - select b from CSVImportConfig b + select b from ImportConfig b where b.name = :name and b.user.username = :username """; - return entityManager.blocking() + return entityManager.blocking() .hql(hql) .set("name", name) .set("username", authenticationFacade.authenticated()) @@ -48,17 +48,17 @@ public Sequence lookup() { log.trace("CSVConfiguration listing"); var hql = """ - select b from CSVImportConfig b + select b from ImportConfig b where b.user.username = :username"""; - return entityManager.blocking() + return entityManager.blocking() .hql(hql) .set("username", authenticationFacade.authenticated()) .sequence() .map(this::convert); } - private BatchImportConfig convert(CSVImportConfig source) { + private BatchImportConfig convert(ImportConfig source) { if (source == null) { return null; } diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/CSVImportConfig.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportConfig.java similarity index 74% rename from jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/CSVImportConfig.java rename to jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportConfig.java index 9a9178fd..ea9a13dc 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/CSVImportConfig.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportConfig.java @@ -15,24 +15,28 @@ @Entity @Getter @Table(name = "import_config") -public class CSVImportConfig extends EntityJpa { +public class ImportConfig extends EntityJpa { private String name; @Column private String fileCode; + @Column + private String type; + @ManyToOne @JoinColumn private UserAccountJpa user; @Builder - private CSVImportConfig(String name, String fileCode, UserAccountJpa user) { + private ImportConfig(String name, String fileCode, String type, UserAccountJpa user) { this.name = name; this.fileCode = fileCode; this.user = user; + this.type = type; } - protected CSVImportConfig() { + protected ImportConfig() { } } diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportJpa.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportJpa.java index 8ee6488f..74b13083 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportJpa.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportJpa.java @@ -25,7 +25,7 @@ public class ImportJpa extends EntityJpa { @ManyToOne @JoinColumn - private CSVImportConfig config; + private ImportConfig config; @ManyToOne @JoinColumn @@ -40,7 +40,7 @@ private ImportJpa( Date finished, String slug, String fileCode, - CSVImportConfig config, + ImportConfig config, UserAccountJpa user, boolean archived, List transactions) { diff --git a/jpa-repository/src/main/resources/db/migration/V20240404131810__add_import_type.sql b/jpa-repository/src/main/resources/db/migration/V20240404131810__add_import_type.sql new file mode 100644 index 00000000..85286b2a --- /dev/null +++ b/jpa-repository/src/main/resources/db/migration/V20240404131810__add_import_type.sql @@ -0,0 +1,5 @@ +-- add the type column and default all existing to CSVImportProvider + +ALTER TABLE import_config + ADD COLUMN type VARCHAR(255) DEFAULT 'CSVImportProvider' NOT NULL; + diff --git a/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigEventListenerIT.java b/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigEventListenerIT.java index 4e5e358a..c540cbf0 100644 --- a/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigEventListenerIT.java +++ b/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigEventListenerIT.java @@ -1,7 +1,7 @@ package com.jongsoft.finance.jpa.importer; import com.jongsoft.finance.jpa.JpaTestSetup; -import com.jongsoft.finance.jpa.importer.entity.CSVImportConfig; +import com.jongsoft.finance.jpa.importer.entity.ImportConfig; import com.jongsoft.finance.messaging.commands.importer.CreateConfigurationCommand; import com.jongsoft.finance.security.AuthenticationFacade; import io.micronaut.context.event.ApplicationEventPublisher; @@ -38,10 +38,10 @@ void setup() { @Test void handleCreatedEvent() { eventPublisher.publishEvent( - new CreateConfigurationCommand("test-config", "file-code-3")); + new CreateConfigurationCommand("CSVImportProvider", "test-config", "file-code-3")); - var query = entityManager.createQuery("select c from CSVImportConfig c where c.name = 'test-config'"); - var check = (CSVImportConfig) query.getSingleResult(); + var query = entityManager.createQuery("select c from ImportConfig c where c.name = 'test-config'"); + var check = (ImportConfig) query.getSingleResult(); Assertions.assertThat(check.getName()).isEqualTo("test-config"); Assertions.assertThat(check.getFileCode()).isEqualTo("file-code-3"); diff --git a/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpaIT.java b/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpaIT.java index b42dcdd0..62756682 100644 --- a/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpaIT.java +++ b/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpaIT.java @@ -1,7 +1,7 @@ package com.jongsoft.finance.jpa.importer; import com.jongsoft.finance.jpa.JpaTestSetup; -import com.jongsoft.finance.providers.CSVConfigProvider; +import com.jongsoft.finance.providers.ImportConfigurationProvider; import com.jongsoft.finance.security.AuthenticationFacade; import io.micronaut.test.annotation.MockBean; import jakarta.inject.Inject; @@ -16,7 +16,7 @@ class CSVConfigProviderJpaIT extends JpaTestSetup { private AuthenticationFacade authenticationFacade; @Inject - private CSVConfigProvider csvConfigProvider; + private ImportConfigurationProvider csvConfigProvider; @BeforeEach void setup() { diff --git a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImporterProvider.java b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImporterProvider.java index add2f5c8..43bbecdf 100644 --- a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImporterProvider.java +++ b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImporterProvider.java @@ -13,4 +13,8 @@ public interface ImporterProvider { boolean supports(X configuration); + default String getImporterType() { + return this.getClass().getSimpleName(); + } + } diff --git a/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java b/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java index 7d74ad2b..35588864 100644 --- a/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java +++ b/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java @@ -126,6 +126,12 @@ void readTransactions_withdrawal() throws IOException { Assertions.assertThat(transaction.type()).isEqualTo(TransactionType.CREDIT); } + @Test + void exposesCorrectType() { + Assertions.assertThat(csvImportProvider.getImporterType()) + .isEqualTo("CSVImportProvider"); + } + private BatchImport createBatchImport() { return BatchImport.builder() .fileCode("my-secret-import-files") From 980af77c57d45d17e0c9f6d7c43ecf6d96294dec Mon Sep 17 00:00:00 2001 From: Gerben Jongerius Date: Thu, 4 Apr 2024 17:23:37 +0200 Subject: [PATCH 5/8] Fix issues in serialization between front-end and backend application. --- .../main/java/com/jongsoft/finance/ProcessVariable.java | 2 +- .../finance/bpmn/ProcessEngineConfiguration.java | 3 ++- .../finance/bpmn/camunda/JsonRecordSerializer.java | 9 ++++++++- .../bpmn/delegate/importer/ExtractionMapping.java | 4 +++- .../jongsoft/finance/serialized/ImportJobSettings.java | 3 ++- bpmn-process/src/test/resources/logback.xml | 2 +- .../rest/importer/CSVImporterConfigCreateRequest.java | 6 ++++++ .../finance/rest/model/CSVImporterConfigResponse.java | 9 +++++++-- .../com/jongsoft/finance/rest/process/VariableMap.java | 5 ++++- fintrack-api/src/main/resources/docs/index.html | 6 +++--- .../jpa/importer/ImportConfigurationProviderJpa.java | 1 + .../jongsoft/finance/jpa/importer/ImportProviderJpa.java | 1 + .../finance/jpa/importer/CSVConfigEventListenerIT.java | 1 + .../finance/jpa/importer/CSVConfigProviderJpaIT.java | 5 ++++- rule-engine/build.gradle.kts | 8 ++++---- .../com/jongsoft/finance/rule/impl/RuleEngineImpl.java | 7 ++++++- .../jongsoft/finance/importer/csv/CSVConfiguration.java | 4 ---- 17 files changed, 54 insertions(+), 22 deletions(-) diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/ProcessVariable.java b/bpmn-process/src/main/java/com/jongsoft/finance/ProcessVariable.java index ce4bf6cb..ee62ce5e 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/ProcessVariable.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/ProcessVariable.java @@ -5,5 +5,5 @@ import java.io.Serializable; @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "_type") -public interface ProcessVariable extends Serializable { +public interface ProcessVariable { } diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java index 87a54964..9d263c2f 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java @@ -1,5 +1,6 @@ package com.jongsoft.finance.bpmn; +import com.jongsoft.finance.ProcessVariable; import com.jongsoft.finance.bpmn.camunda.*; import com.jongsoft.finance.core.DataSourceMigration; import com.jongsoft.finance.serialized.ImportJobSettings; @@ -57,7 +58,7 @@ public ProcessEngine processEngine() throws IOException { configuration.setHistoryTimeToLive("P1D"); configuration.setResolverFactories(List.of(new MicronautBeanResolver(applicationContext))); configuration.setCustomPreVariableSerializers(List.of( - new JsonRecordSerializer<>(applicationContext.getBean(ObjectMapper.class), ImportJobSettings.class))); + new JsonRecordSerializer<>(applicationContext.getBean(ObjectMapper.class), ProcessVariable.class))); var processEngine = configuration.buildProcessEngine(); log.info("Created camunda process engine"); diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/camunda/JsonRecordSerializer.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/camunda/JsonRecordSerializer.java index 8e18fd65..b474e3e9 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/camunda/JsonRecordSerializer.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/camunda/JsonRecordSerializer.java @@ -8,10 +8,13 @@ import org.camunda.bpm.engine.variable.impl.value.UntypedValueImpl; import org.camunda.bpm.engine.variable.value.ObjectValue; import org.camunda.bpm.engine.variable.value.TypedValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; public class JsonRecordSerializer extends AbstractTypedValueSerializer { + private static final Logger logger = LoggerFactory.getLogger(JsonRecordSerializer.class); final ObjectMapper objectMapper; final Class supportedClass; @@ -34,6 +37,8 @@ public String getSerializationDataformat() { @Override public TypedValue convertToTypedValue(UntypedValueImpl untypedValue) { + logger.debug("Converting untyped value to typed value: {}", untypedValue.getType()); + var importJobSettings = (Record) untypedValue.getValue(); String jsonString; try { @@ -54,6 +59,7 @@ public void writeValue(TypedValue typedValue, ValueFields valueFields) { @Override public TypedValue readValue(ValueFields valueFields, boolean b, boolean b1) { + logger.debug("Reading value from value fields: {}", valueFields.getName()); try { return Variables.objectValue(objectMapper.readValue( new String(valueFields.getByteArrayValue()), @@ -67,6 +73,7 @@ public TypedValue readValue(ValueFields valueFields, boolean b, boolean b1) { @Override protected boolean canWriteValue(TypedValue typedValue) { - return supportedClass.isAssignableFrom(typedValue.getValue().getClass()); + logger.trace("Checking if value can be written: {}", typedValue.getValue().getClass().getSimpleName()); + return supportedClass.isInstance(typedValue.getValue()); } } diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractionMapping.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractionMapping.java index 73565c29..66f65fb8 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractionMapping.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractionMapping.java @@ -5,10 +5,12 @@ import lombok.EqualsAndHashCode; import lombok.Getter; +import java.io.Serializable; + @Getter @Serdeable @EqualsAndHashCode(of = {"name"}) -public class ExtractionMapping implements ProcessVariable { +public class ExtractionMapping implements ProcessVariable, Serializable { private final String name; private final Long accountId; diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportJobSettings.java b/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportJobSettings.java index 41704462..59ad963a 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportJobSettings.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportJobSettings.java @@ -1,6 +1,7 @@ package com.jongsoft.finance.serialized; import com.fasterxml.jackson.annotation.JsonProperty; +import com.jongsoft.finance.ProcessVariable; import com.jongsoft.finance.importer.api.ImporterConfiguration; import io.micronaut.serde.annotation.Serdeable; @@ -13,4 +14,4 @@ public record ImportJobSettings( @JsonProperty boolean generateAccounts, @JsonProperty - Long accountId) {} + Long accountId) implements ProcessVariable {} diff --git a/bpmn-process/src/test/resources/logback.xml b/bpmn-process/src/test/resources/logback.xml index 423f1e4f..3ef0b7ce 100644 --- a/bpmn-process/src/test/resources/logback.xml +++ b/bpmn-process/src/test/resources/logback.xml @@ -9,7 +9,7 @@ - + diff --git a/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/CSVImporterConfigCreateRequest.java b/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/CSVImporterConfigCreateRequest.java index b2b2a871..7efc7b2d 100644 --- a/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/CSVImporterConfigCreateRequest.java +++ b/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/CSVImporterConfigCreateRequest.java @@ -1,14 +1,20 @@ package com.jongsoft.finance.rest.importer; import io.micronaut.serde.annotation.Serdeable; +import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotBlank; @Serdeable.Deserializable record CSVImporterConfigCreateRequest( @NotBlank + @Schema(description = "The type of importer that is to be used") + String type, + @Schema(description = "The name of the configuration") + @NotBlank String name, + @Schema(description = "The file code to get the contents of the configuration") @NotBlank String fileCode) { } diff --git a/fintrack-api/src/main/java/com/jongsoft/finance/rest/model/CSVImporterConfigResponse.java b/fintrack-api/src/main/java/com/jongsoft/finance/rest/model/CSVImporterConfigResponse.java index 41b308b1..8512c707 100644 --- a/fintrack-api/src/main/java/com/jongsoft/finance/rest/model/CSVImporterConfigResponse.java +++ b/fintrack-api/src/main/java/com/jongsoft/finance/rest/model/CSVImporterConfigResponse.java @@ -13,16 +13,21 @@ public CSVImporterConfigResponse(BatchImportConfig wrapped) { this.wrapped = wrapped; } - @Schema(description = "The CSV configuration identifier") + @Schema(description = "The configuration identifier") public Long getId() { return wrapped.getId(); } - @Schema(description = "The name of the CSV configuration") + @Schema(description = "The name of the configuration") public String getName() { return wrapped.getName(); } + @Schema(description = "The type of importer that will be used") + public String getType() { + return wrapped.getType(); + } + @Schema(description = "The file code to get the contents of the configuration") public String getFile() { return wrapped.getFileCode(); diff --git a/fintrack-api/src/main/java/com/jongsoft/finance/rest/process/VariableMap.java b/fintrack-api/src/main/java/com/jongsoft/finance/rest/process/VariableMap.java index acd9cf4c..cc528dbb 100644 --- a/fintrack-api/src/main/java/com/jongsoft/finance/rest/process/VariableMap.java +++ b/fintrack-api/src/main/java/com/jongsoft/finance/rest/process/VariableMap.java @@ -1,5 +1,6 @@ package com.jongsoft.finance.rest.process; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.jongsoft.finance.ProcessVariable; import io.micronaut.serde.annotation.Serdeable; import io.swagger.v3.oas.annotations.media.Schema; @@ -19,7 +20,9 @@ public record VariableList(List content) implements ProcessVari @Serdeable @Schema(name = "WrappedVariable", description = "A variable wrapped for the task.") - public record WrappedVariable(T value) implements ProcessVariable { + public record WrappedVariable( + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "_type") + T value) implements ProcessVariable { } @Schema(description = "The actual map of all the variables set for the task.") diff --git a/fintrack-api/src/main/resources/docs/index.html b/fintrack-api/src/main/resources/docs/index.html index 010e18d1..425bd883 100644 --- a/fintrack-api/src/main/resources/docs/index.html +++ b/fintrack-api/src/main/resources/docs/index.html @@ -3,7 +3,7 @@ - FinTrack: REST API Documentation + Pledger.io: REST API Documentation @@ -24,10 +24,10 @@ allow-server-selection="false">
-
FinTrack API
+
Pledger.io API

- FinTrack is a "self hosted" application that helps in keeping track of your personal finances. + Pledger.io is a "self hosted" application that helps in keeping track of your personal finances. It helps you keep track of your income and expenses to allow you to spend less many and save more.

diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportConfigurationProviderJpa.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportConfigurationProviderJpa.java index b6042e5c..80646dbb 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportConfigurationProviderJpa.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportConfigurationProviderJpa.java @@ -66,6 +66,7 @@ private BatchImportConfig convert(ImportConfig source) { return BatchImportConfig.builder() .id(source.getId()) .name(source.getName()) + .type(source.getType()) .fileCode(source.getFileCode()) .user(UserAccount.builder() .id(source.getUser().getId()) diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportProviderJpa.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportProviderJpa.java index 46c768b0..a696018f 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportProviderJpa.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportProviderJpa.java @@ -72,6 +72,7 @@ protected BatchImport convert(ImportJpa source) { .slug(source.getSlug()) .finished(source.getFinished()) .config(BatchImportConfig.builder() + .type(source.getConfig().getType()) .name(source.getConfig().getName()) .fileCode(source.getConfig().getFileCode()) .build()) diff --git a/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigEventListenerIT.java b/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigEventListenerIT.java index c540cbf0..a61f1593 100644 --- a/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigEventListenerIT.java +++ b/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigEventListenerIT.java @@ -45,6 +45,7 @@ void handleCreatedEvent() { Assertions.assertThat(check.getName()).isEqualTo("test-config"); Assertions.assertThat(check.getFileCode()).isEqualTo("file-code-3"); + Assertions.assertThat(check.getType()).isEqualTo("CSVImportProvider"); } @MockBean diff --git a/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpaIT.java b/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpaIT.java index 62756682..866429b0 100644 --- a/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpaIT.java +++ b/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpaIT.java @@ -33,7 +33,10 @@ void lookup() { Assertions.assertThat(csvConfigProvider.lookup()) .hasSize(1) .first() - .satisfies(batch -> Assertions.assertThat(batch.getFileCode()).isEqualTo("file-code-1")); + .satisfies(batch -> { + Assertions.assertThat(batch.getFileCode()).isEqualTo("file-code-1"); + Assertions.assertThat(batch.getType()).isEqualTo("CSVImportProvider"); + }); } @Test diff --git a/rule-engine/build.gradle.kts b/rule-engine/build.gradle.kts index 6c5bc1e5..c5bc9c6a 100644 --- a/rule-engine/build.gradle.kts +++ b/rule-engine/build.gradle.kts @@ -1,11 +1,11 @@ -plugins { - id("io.freefair.lombok") -} - dependencies { + annotationProcessor(mn.lombok) + implementation(libs.lang) implementation(project(":core")) implementation(project(":domain")) + compileOnly(mn.lombok) + testImplementation(libs.bundles.junit) } diff --git a/rule-engine/src/main/java/com/jongsoft/finance/rule/impl/RuleEngineImpl.java b/rule-engine/src/main/java/com/jongsoft/finance/rule/impl/RuleEngineImpl.java index c505766a..1f8669e7 100644 --- a/rule-engine/src/main/java/com/jongsoft/finance/rule/impl/RuleEngineImpl.java +++ b/rule-engine/src/main/java/com/jongsoft/finance/rule/impl/RuleEngineImpl.java @@ -16,12 +16,17 @@ import java.util.List; @Singleton -@RequiredArgsConstructor(onConstructor_ = @Inject) public class RuleEngineImpl implements RuleEngine { private final TransactionRuleProvider transactionRuleProvider; private final List locators; + @Inject + public RuleEngineImpl(TransactionRuleProvider transactionRuleProvider, List locators) { + this.transactionRuleProvider = transactionRuleProvider; + this.locators = locators; + } + @Override public RuleDataSet run(RuleDataSet input) { var outputSet = new RuleDataSet(); diff --git a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVConfiguration.java b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVConfiguration.java index 4fb7bf5e..e58f3f8d 100644 --- a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVConfiguration.java +++ b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVConfiguration.java @@ -10,10 +10,6 @@ public record CSVConfiguration( @JsonProperty("has-headers") boolean headers, - @JsonProperty("apply-rules") - boolean applyRules, - @JsonProperty("generate-accounts") - boolean generateAccounts, @JsonProperty("date-format") String dateFormat, @JsonProperty("delimiter") From c30635f5067e9d25c5e23657755f06cee195d0b2 Mon Sep 17 00:00:00 2001 From: Gerben Jongerius Date: Fri, 5 Apr 2024 15:43:34 +0200 Subject: [PATCH 6/8] Correct a broken test. --- .../jongsoft/finance/importer/csv/CSVImportProviderTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java b/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java index 35588864..994c60e6 100644 --- a/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java +++ b/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java @@ -60,12 +60,10 @@ void loadConfiguration() throws IOException { Assertions.assertThat(configuration) .isNotNull() - .extracting(CSVConfiguration::delimiter, CSVConfiguration::headers, CSVConfiguration::applyRules, CSVConfiguration::generateAccounts, CSVConfiguration::dateFormat, CSVConfiguration::transactionTypeIndicator, CSVConfiguration::columnRoles) + .extracting(CSVConfiguration::delimiter, CSVConfiguration::headers, CSVConfiguration::dateFormat, CSVConfiguration::transactionTypeIndicator, CSVConfiguration::columnRoles) .isEqualTo(List.of( ',', true, - true, - false, "yyyyMMdd", new TransactionTypeIndicator("Bij", "Af"), List.of( From 5442380ad177af78d564e336e1c7da74bbfcd2a2 Mon Sep 17 00:00:00 2001 From: Gerben Jongerius Date: Fri, 5 Apr 2024 16:17:42 +0200 Subject: [PATCH 7/8] Add early exception when the import job yields no transactions. --- .../delegate/importer/ExtractAccountDetailsDelegate.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java index 16a6f3ef..bbf51a04 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java @@ -1,5 +1,6 @@ package com.jongsoft.finance.bpmn.delegate.importer; +import com.jongsoft.finance.core.exception.StatusException; import com.jongsoft.finance.importer.ImporterProvider; import com.jongsoft.finance.importer.api.ImporterConfiguration; import com.jongsoft.finance.providers.ImportProvider; @@ -53,6 +54,11 @@ public void execute(DelegateExecution execution) throws Exception { () -> logger.warn("No importer provider found for configuration: {}", importJobSettings.importConfiguration()) ); + if (locatable.isEmpty()) { + logger.warn("No accounts found for import job {}", batchImportSlug); + throw StatusException.internalError("No parsable accounts found for import job", "bpmn.transaction.import.no-accounts-found"); + } + execution.setVariableLocal("locatable", locatable); } } From 62b08e9e21ae8e629bf4c24dde50761e6e0f7b26 Mon Sep 17 00:00:00 2001 From: Gerben Jongerius Date: Fri, 5 Apr 2024 16:23:55 +0200 Subject: [PATCH 8/8] Add some documentation to the importer delegates for BPMN processes. --- .../ExtractAccountDetailsDelegate.java | 17 ++++++++++------ .../delegate/importer/ExtractionMapping.java | 3 +++ .../importer/LoadImporterConfiguration.java | 20 ++++++++++--------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java index bbf51a04..d3baa87f 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java @@ -17,6 +17,13 @@ import java.util.List; import java.util.Set; +/** + * Fetches the transaction details from the import job and extracts the information needed to locate the account. + *

+ * The extracted information is stored in the {@code locatable} variable in the execution context. + * The locatable contains a set of {@link ExtractedAccountLookup} objects that can be used to locate the account. + *

+ */ @Singleton public class ExtractAccountDetailsDelegate implements JavaDelegate { private final static Logger logger = LoggerFactory.getLogger(ExtractAccountDetailsDelegate.class); @@ -43,12 +50,10 @@ public void execute(DelegateExecution execution) throws Exception { .findFirst() .ifPresentOrElse( provider -> provider.readTransactions( - transactionDTO -> { - locatable.add(new ExtractedAccountLookup( - transactionDTO.opposingName(), - transactionDTO.opposingIBAN(), - transactionDTO.description())); - }, + transactionDTO -> locatable.add(new ExtractedAccountLookup( + transactionDTO.opposingName(), + transactionDTO.opposingIBAN(), + transactionDTO.description())), importJobSettings.importConfiguration(), importJob), () -> logger.warn("No importer provider found for configuration: {}", importJobSettings.importConfiguration()) diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractionMapping.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractionMapping.java index 66f65fb8..6efbeb87 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractionMapping.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractionMapping.java @@ -7,6 +7,9 @@ import java.io.Serializable; +/** + * Represents a mapping between an account name and an account ID. + */ @Getter @Serdeable @EqualsAndHashCode(of = {"name"}) diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java index c27616ec..d3552b44 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java @@ -10,6 +10,10 @@ import java.util.List; +/** + * Loads the importer configuration for the given batch import. + * The importer configuration is loaded from the importer provider that matches the import type and is stored in the {@code importConfig} variable. + **/ @Slf4j @Singleton public class LoadImporterConfiguration implements JavaDelegate { @@ -37,15 +41,13 @@ public void execute(DelegateExecution delegateExecution) throws Exception { .filter(importer -> importer.getImporterType().equalsIgnoreCase(importJob.getConfig().getType())) .findFirst() .ifPresentOrElse( - importer -> { - delegateExecution.setVariableLocal( - "importConfig", - new ImportJobSettings( - importer.loadConfiguration(importJob.getConfig()), - false, - false, - null)); - }, + importer -> delegateExecution.setVariableLocal( + "importConfig", + new ImportJobSettings( + importer.loadConfiguration(importJob.getConfig()), + false, + false, + null)), () -> { throw new IllegalStateException("Cannot find importer for type " + importJob.getConfig().getType()); }