diff --git a/README.md b/README.md index 1f934fc9..f043fd3b 100644 --- a/README.md +++ b/README.md @@ -48,20 +48,3 @@ API documentation use the url: http://localhost:8080/spec/index.html -## License -Copyright 2024 Jong Soft Development - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and -associated documentation files (the "Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial -portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT -LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE -OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/bpmn-process/build.gradle.kts b/bpmn-process/build.gradle.kts index 0d202b1e..74373b74 100644 --- a/bpmn-process/build.gradle.kts +++ b/bpmn-process/build.gradle.kts @@ -23,6 +23,7 @@ dependencies { implementation(project(":core")) implementation(project(":domain")) implementation(project(":rule-engine")) + implementation(project(":transaction-importer:transaction-importer-api")) // needed for the testing of the application runtimeOnly(mn.h2) @@ -30,4 +31,5 @@ dependencies { testRuntimeOnly(mn.logback.classic) testImplementation(mn.micronaut.test.junit5) testImplementation(libs.bundles.junit) + testRuntimeOnly(project(":transaction-importer:transaction-importer-csv")) } diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/ProcessVariable.java b/bpmn-process/src/main/java/com/jongsoft/finance/ProcessVariable.java index ce4bf6cb..ee62ce5e 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/ProcessVariable.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/ProcessVariable.java @@ -5,5 +5,5 @@ import java.io.Serializable; @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "_type") -public interface ProcessVariable extends Serializable { +public interface ProcessVariable { } diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java index 6f06d9aa..9d263c2f 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/ProcessEngineConfiguration.java @@ -1,11 +1,14 @@ package com.jongsoft.finance.bpmn; +import com.jongsoft.finance.ProcessVariable; import com.jongsoft.finance.bpmn.camunda.*; import com.jongsoft.finance.core.DataSourceMigration; +import com.jongsoft.finance.serialized.ImportJobSettings; import io.micronaut.context.ApplicationContext; import io.micronaut.context.annotation.Context; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; +import io.micronaut.serde.ObjectMapper; import lombok.extern.slf4j.Slf4j; import org.camunda.bpm.engine.HistoryService; import org.camunda.bpm.engine.ProcessEngine; @@ -54,6 +57,8 @@ public ProcessEngine processEngine() throws IOException { configuration.setHistoryCleanupBatchWindowEndTime("03:00"); configuration.setHistoryTimeToLive("P1D"); configuration.setResolverFactories(List.of(new MicronautBeanResolver(applicationContext))); + configuration.setCustomPreVariableSerializers(List.of( + new JsonRecordSerializer<>(applicationContext.getBean(ObjectMapper.class), ProcessVariable.class))); var processEngine = configuration.buildProcessEngine(); log.info("Created camunda process engine"); diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/camunda/JsonRecordSerializer.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/camunda/JsonRecordSerializer.java new file mode 100644 index 00000000..b474e3e9 --- /dev/null +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/camunda/JsonRecordSerializer.java @@ -0,0 +1,79 @@ +package com.jongsoft.finance.bpmn.camunda; + +import io.micronaut.serde.ObjectMapper; +import org.camunda.bpm.engine.impl.variable.serializer.AbstractTypedValueSerializer; +import org.camunda.bpm.engine.impl.variable.serializer.ValueFields; +import org.camunda.bpm.engine.variable.Variables; +import org.camunda.bpm.engine.variable.impl.type.ObjectTypeImpl; +import org.camunda.bpm.engine.variable.impl.value.UntypedValueImpl; +import org.camunda.bpm.engine.variable.value.ObjectValue; +import org.camunda.bpm.engine.variable.value.TypedValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class JsonRecordSerializer extends AbstractTypedValueSerializer { + private static final Logger logger = LoggerFactory.getLogger(JsonRecordSerializer.class); + + final ObjectMapper objectMapper; + final Class supportedClass; + + public JsonRecordSerializer(ObjectMapper objectMapper, Class supportedClass) { + super(new ObjectTypeImpl()); + this.objectMapper = objectMapper; + this.supportedClass = supportedClass; + } + + @Override + public String getName() { + return "record-json"; + } + + @Override + public String getSerializationDataformat() { + return getName(); + } + + @Override + public TypedValue convertToTypedValue(UntypedValueImpl untypedValue) { + logger.debug("Converting untyped value to typed value: {}", untypedValue.getType()); + + var importJobSettings = (Record) untypedValue.getValue(); + String jsonString; + try { + jsonString = objectMapper.writeValueAsString(importJobSettings); + } catch (IOException e) { + throw new RuntimeException("Could not serialize ImportJobSettings", e); + } + return Variables.serializedObjectValue(jsonString) + .serializationDataFormat(getName()) + .create(); + } + + @Override + public void writeValue(TypedValue typedValue, ValueFields valueFields) { + ObjectValue objectValue = (ObjectValue) typedValue; + valueFields.setByteArrayValue(objectValue.getValueSerialized().getBytes()); + } + + @Override + public TypedValue readValue(ValueFields valueFields, boolean b, boolean b1) { + logger.debug("Reading value from value fields: {}", valueFields.getName()); + try { + return Variables.objectValue(objectMapper.readValue( + new String(valueFields.getByteArrayValue()), + supportedClass)) + .serializationDataFormat(getName()) + .create(); + } catch (IOException e) { + throw new RuntimeException("Could not deserialize ImportJobSettings", e); + } + } + + @Override + protected boolean canWriteValue(TypedValue typedValue) { + logger.trace("Checking if value can be written: {}", typedValue.getValue().getClass().getSimpleName()); + return supportedClass.isInstance(typedValue.getValue()); + } +} diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/CSVReaderDelegate.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/CSVReaderDelegate.java deleted file mode 100644 index a4a4f404..00000000 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/CSVReaderDelegate.java +++ /dev/null @@ -1,205 +0,0 @@ -package com.jongsoft.finance.bpmn.delegate.importer; - -import com.jongsoft.finance.ProcessMapper; -import com.jongsoft.finance.StorageService; -import com.jongsoft.finance.domain.transaction.Transaction; -import com.jongsoft.finance.providers.ImportProvider; -import com.jongsoft.finance.serialized.ImportConfigJson; -import com.opencsv.CSVParserBuilder; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvValidationException; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.camunda.bpm.engine.delegate.DelegateExecution; -import org.camunda.bpm.engine.delegate.JavaDelegate; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.EnumMap; -import java.util.Optional; - -@Slf4j -abstract class CSVReaderDelegate implements JavaDelegate { - - private final ImportProvider importProvider; - private final StorageService storageService; - - private final ProcessMapper mapper; - - protected CSVReaderDelegate(ImportProvider importProvider, StorageService storageService, ProcessMapper mapper) { - this.importProvider = importProvider; - this.storageService = storageService; - this.mapper = mapper; - } - - @Override - public void execute(DelegateExecution execution) throws IOException, CsvValidationException { - var batchImportSlug = (String) execution.getVariableLocal("batchImportSlug"); - var importConfigJson = getFromContext(execution); - - var batchImport = importProvider.lookup(batchImportSlug).get(); - log.debug("{}: Processing transaction import CSV {}", execution.getCurrentActivityName(), batchImport.getSlug()); - if (importConfigJson == null) { - throw new IllegalStateException("Cannot run account extraction without actual configuration."); - } - - var mappingIndices = computeIndices(importConfigJson.getColumnRoles()); - - beforeProcess(execution, importConfigJson); - - try (var inputStream = storageService.read(batchImport.getFileCode()) - .map(bytes -> new InputStreamReader(new ByteArrayInputStream(bytes))) - .get()) { - var reader = new CSVReaderBuilder(inputStream) - .withCSVParser(new CSVParserBuilder() - .withSeparator(importConfigJson.getDelimiter()) - .build()) - .build(); - - if (importConfigJson.isHeaders()) { - reader.skip(1); - } - - var line = reader.readNext(); - while (line != null) { - if (lineMatcherRequirements(line, mappingIndices.size())) { - var transaction = transform(line, mappingIndices, importConfigJson); - lineRead(execution, transaction); - } - line = reader.readNext(); - } - } - - afterProcess(execution); - } - - protected abstract void beforeProcess(DelegateExecution execution, ImportConfigJson configJson); - - protected abstract void lineRead(DelegateExecution execution, ParsedTransaction parsedTransaction); - - protected abstract void afterProcess(DelegateExecution execution); - - private boolean lineMatcherRequirements(String[] line, int mappingIndicesCount) { - if (line.length == 1) { - log.debug("Invalid short line in CSV {}", String.join(", ", line)); - } else if (line.length < mappingIndicesCount) { - log.debug("Line contains to few columns got {} expected {}, `{}`", - line.length, - mappingIndicesCount, - String.join(", ", line)); - } else { - return true; - } - - return false; - } - - private ImportConfigJson getFromContext(DelegateExecution execution) { - var rawEntity = execution.getVariable("importConfig"); - - return switch (rawEntity) { - case String json -> mapper.readSafe(json, ImportConfigJson.class); - case ImportConfigJson json -> json; - default -> throw new IllegalArgumentException("Unsupported import configuration provided."); - }; - } - - private ParsedTransaction transform( - String[] csvLine, - EnumMap mappings, - ImportConfigJson configJson) { - var amount = parseAmount(mappings, csvLine); - var date = parseDate( - mappings, - csvLine, - configJson.getDateFormat(), - ImportConfigJson.MappingRole.DATE); - var bookDate = parseDate( - mappings, - csvLine, - configJson.getDateFormat(), - ImportConfigJson.MappingRole.BOOK_DATE); - var interestDate = parseDate( - mappings, - csvLine, - configJson.getDateFormat(), - ImportConfigJson.MappingRole.INTEREST_DATE); - var opposingName = locateColumn(csvLine, mappings, ImportConfigJson.MappingRole.OPPOSING_NAME); - var opposingIBAN = locateColumn(csvLine, mappings, ImportConfigJson.MappingRole.OPPOSING_IBAN); - var description = locateColumn(csvLine, mappings, ImportConfigJson.MappingRole.DESCRIPTION); - - Transaction.Type type = amount >= 0 ? Transaction.Type.DEBIT : Transaction.Type.CREDIT; - if (mappings.containsKey(ImportConfigJson.MappingRole.CUSTOM_INDICATOR)) { - type = parseType(csvLine, mappings, configJson.getCustomIndicator()); - } - - return new ParsedTransaction( - amount, - type, - description, - date, - interestDate, - bookDate, - opposingIBAN, - opposingName); - } - - private EnumMap computeIndices( - Iterable columnRoles) { - EnumMap mapConfig = - new EnumMap<>(ImportConfigJson.MappingRole.class); - - var i = 0; - for (ImportConfigJson.MappingRole role : columnRoles) { - mapConfig.put(role, i); - i++; - } - - return mapConfig; - } - - private LocalDate parseDate( - EnumMap mappingIndices, String[] line, - String format, ImportConfigJson.MappingRole role) { - return Optional.ofNullable(locateColumn(line, mappingIndices, role)) - .map(str -> LocalDate.parse(str, DateTimeFormatter.ofPattern(format))) - .orElse(null); - } - - private Transaction.Type parseType( - String[] line, EnumMap mappingIndices, - ImportConfigJson.CustomIndicator customIndicator) { - return Optional.ofNullable(locateColumn(line, mappingIndices, ImportConfigJson.MappingRole.CUSTOM_INDICATOR)) - .map(str -> { - if (customIndicator.getCredit().equalsIgnoreCase(str)) { - return Transaction.Type.CREDIT; - } else { - return Transaction.Type.DEBIT; - } - }).orElseThrow(() -> new IllegalStateException("Incorrect custom indicator found")); - } - - private double parseAmount( - EnumMap mappingIndices, - String[] line) { - String amount = locateColumn(line, mappingIndices, ImportConfigJson.MappingRole.AMOUNT); - if (StringUtils.isEmpty(amount)) { - throw new IllegalStateException("Amount cannot be blank in import"); - } - - return Double.parseDouble(amount.replace(',', '.')); - } - - private String locateColumn( - String[] line, EnumMap mappingIndices, - ImportConfigJson.MappingRole column) { - if (mappingIndices.containsKey(column)) { - return line[mappingIndices.get(column)]; - } - - return null; - } -} diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java index 215afab6..d3baa87f 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractAccountDetailsDelegate.java @@ -1,46 +1,69 @@ package com.jongsoft.finance.bpmn.delegate.importer; -import com.jongsoft.finance.ProcessMapper; -import com.jongsoft.finance.StorageService; +import com.jongsoft.finance.core.exception.StatusException; +import com.jongsoft.finance.importer.ImporterProvider; +import com.jongsoft.finance.importer.api.ImporterConfiguration; import com.jongsoft.finance.providers.ImportProvider; import com.jongsoft.finance.serialized.ExtractedAccountLookup; -import com.jongsoft.finance.serialized.ImportConfigJson; +import com.jongsoft.finance.serialized.ImportJobSettings; import jakarta.inject.Inject; import jakarta.inject.Singleton; import org.camunda.bpm.engine.delegate.DelegateExecution; +import org.camunda.bpm.engine.delegate.JavaDelegate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.HashSet; +import java.util.List; +import java.util.Set; +/** + * Fetches the transaction details from the import job and extracts the information needed to locate the account. + *

+ * The extracted information is stored in the {@code locatable} variable in the execution context. + * The locatable contains a set of {@link ExtractedAccountLookup} objects that can be used to locate the account. + *

+ */ @Singleton -public class ExtractAccountDetailsDelegate extends CSVReaderDelegate { +public class ExtractAccountDetailsDelegate implements JavaDelegate { + private final static Logger logger = LoggerFactory.getLogger(ExtractAccountDetailsDelegate.class); + + private final List> importerProviders; + private final ImportProvider importProvider; @Inject - public ExtractAccountDetailsDelegate( - ImportProvider importProvider, - StorageService storageService, - ProcessMapper mapper) { - super(importProvider, storageService, mapper); + public ExtractAccountDetailsDelegate(List> importerProviders, ImportProvider importProvider) { + this.importerProviders = importerProviders; + this.importProvider = importProvider; } @Override - protected void beforeProcess(DelegateExecution execution, ImportConfigJson configJson) { - execution.setVariableLocal("locatable", new HashSet<>()); - } + public void execute(DelegateExecution execution) throws Exception { + var batchImportSlug = (String) execution.getVariableLocal("batchImportSlug"); + var importJobSettings = (ImportJobSettings) execution.getVariable("importConfig"); + logger.debug("{}: Processing transaction import {}", execution.getCurrentActivityName(), batchImportSlug); - @Override - protected void lineRead(DelegateExecution execution, ParsedTransaction parsedTransaction) { - var extraction = new ExtractedAccountLookup( - parsedTransaction.getOpposingName(), - parsedTransaction.getOpposingIBAN(), - parsedTransaction.getDescription()); - - @SuppressWarnings("unchecked") - var locatable = (HashSet) execution.getVariableLocal("locatable"); - locatable.add(extraction); - } + var importJob = importProvider.lookup(batchImportSlug).get(); + Set locatable = new HashSet<>(); + importerProviders.stream() + .filter(provider -> provider.supports(importJobSettings.importConfiguration())) + .findFirst() + .ifPresentOrElse( + provider -> provider.readTransactions( + transactionDTO -> locatable.add(new ExtractedAccountLookup( + transactionDTO.opposingName(), + transactionDTO.opposingIBAN(), + transactionDTO.description())), + importJobSettings.importConfiguration(), + importJob), + () -> logger.warn("No importer provider found for configuration: {}", importJobSettings.importConfiguration()) + ); - @Override - protected void afterProcess(DelegateExecution execution) { - execution.setVariable("extractionResult", new HashSet<>()); + if (locatable.isEmpty()) { + logger.warn("No accounts found for import job {}", batchImportSlug); + throw StatusException.internalError("No parsable accounts found for import job", "bpmn.transaction.import.no-accounts-found"); + } + + execution.setVariableLocal("locatable", locatable); } } diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractionMapping.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractionMapping.java index 73565c29..6efbeb87 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractionMapping.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ExtractionMapping.java @@ -5,10 +5,15 @@ import lombok.EqualsAndHashCode; import lombok.Getter; +import java.io.Serializable; + +/** + * Represents a mapping between an account name and an account ID. + */ @Getter @Serdeable @EqualsAndHashCode(of = {"name"}) -public class ExtractionMapping implements ProcessVariable { +public class ExtractionMapping implements ProcessVariable, Serializable { private final String name; private final Long accountId; diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java index 0d676c7f..d3552b44 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/LoadImporterConfiguration.java @@ -1,33 +1,29 @@ package com.jongsoft.finance.bpmn.delegate.importer; -import com.jongsoft.finance.ProcessMapper; -import com.jongsoft.finance.StorageService; +import com.jongsoft.finance.importer.ImporterProvider; import com.jongsoft.finance.providers.ImportProvider; -import com.jongsoft.finance.serialized.ImportConfigJson; -import jakarta.inject.Inject; +import com.jongsoft.finance.serialized.ImportJobSettings; import jakarta.inject.Singleton; import lombok.extern.slf4j.Slf4j; import org.camunda.bpm.engine.delegate.DelegateExecution; import org.camunda.bpm.engine.delegate.JavaDelegate; -import java.nio.charset.StandardCharsets; +import java.util.List; +/** + * Loads the importer configuration for the given batch import. + * The importer configuration is loaded from the importer provider that matches the import type and is stored in the {@code importConfig} variable. + **/ @Slf4j @Singleton public class LoadImporterConfiguration implements JavaDelegate { private final ImportProvider importProvider; - private final StorageService storageService; - private final ProcessMapper mapper; - - @Inject - public LoadImporterConfiguration( - ImportProvider importProvider, - StorageService storageService, - ProcessMapper mapper) { + private final List> importerProvider; + + public LoadImporterConfiguration(ImportProvider importProvider, List> importerProvider) { this.importProvider = importProvider; - this.storageService = storageService; - this.mapper = mapper; + this.importerProvider = importerProvider; } @Override @@ -41,15 +37,20 @@ public void execute(DelegateExecution delegateExecution) throws Exception { var importJob = importProvider.lookup(batchImportSlug) .getOrThrow(() -> new IllegalStateException("Cannot find batch import with slug " + batchImportSlug)); - var importConfig = readConfiguration(importJob.getConfig().getFileCode()); - - delegateExecution.setVariableLocal("importConfig", importConfig); - } - - private ImportConfigJson readConfiguration(String fileCode) { - return storageService.read(fileCode) - .map(bytes -> new String(bytes, StandardCharsets.UTF_8)) - .map(json -> mapper.readSafe(json, ImportConfigJson.class)) - .getOrThrow(() -> new IllegalStateException("Cannot read import configuration from file " + fileCode)); + importerProvider.stream() + .filter(importer -> importer.getImporterType().equalsIgnoreCase(importJob.getConfig().getType())) + .findFirst() + .ifPresentOrElse( + importer -> delegateExecution.setVariableLocal( + "importConfig", + new ImportJobSettings( + importer.loadConfiguration(importJob.getConfig()), + false, + false, + null)), + () -> { + throw new IllegalStateException("Cannot find importer for type " + importJob.getConfig().getType()); + } + ); } } diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ReadTransactionLogDelegate.java b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ReadTransactionLogDelegate.java index b40fa31e..048c9ad2 100644 --- a/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ReadTransactionLogDelegate.java +++ b/bpmn-process/src/main/java/com/jongsoft/finance/bpmn/delegate/importer/ReadTransactionLogDelegate.java @@ -2,8 +2,10 @@ import com.jongsoft.finance.ProcessMapper; import com.jongsoft.finance.StorageService; +import com.jongsoft.finance.importer.ImporterProvider; +import com.jongsoft.finance.importer.api.ImporterConfiguration; import com.jongsoft.finance.providers.ImportProvider; -import com.jongsoft.finance.serialized.ImportConfigJson; +import com.jongsoft.finance.serialized.ImportJobSettings; import jakarta.inject.Inject; import jakarta.inject.Singleton; import lombok.extern.slf4j.Slf4j; @@ -12,46 +14,56 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.List; @Slf4j @Singleton -public class ReadTransactionLogDelegate extends CSVReaderDelegate implements JavaDelegate { +public class ReadTransactionLogDelegate implements JavaDelegate { + private final List> importerProviders; + private final ImportProvider importProvider; private final StorageService storageService; private final ProcessMapper mapper; @Inject - public ReadTransactionLogDelegate(ImportProvider importProvider, StorageService storageService, ProcessMapper mapper) { - super(importProvider, storageService, mapper); + public ReadTransactionLogDelegate( + List> importerProviders, + ImportProvider importProvider, + StorageService storageService, + ProcessMapper mapper) { + this.importerProviders = importerProviders; + this.importProvider = importProvider; this.storageService = storageService; this.mapper = mapper; } @Override - protected void beforeProcess(DelegateExecution execution, ImportConfigJson configJson) { - log.debug("Setting up reader for import file with configuration {}", configJson); - execution.setVariableLocal("generateAccounts", configJson.isGenerateAccounts()); - execution.setVariableLocal("applyRules", configJson.isApplyRules()); - execution.setVariableLocal("targetAccountId", configJson.getAccountId()); - execution.setVariableLocal("storageTokens", new ArrayList()); - } - - @Override - protected void lineRead(DelegateExecution execution, ParsedTransaction parsedTransaction) { - log.debug("Read line {} of file import", parsedTransaction); + public void execute(DelegateExecution execution) throws Exception { + var batchImportSlug = (String) execution.getVariableLocal("batchImportSlug"); + var importJobSettings = (ImportJobSettings) execution.getVariable("importConfig"); + log.debug("{}: Processing transaction import {}", execution.getCurrentActivityName(), batchImportSlug); - var serialized = mapper.writeSafe(parsedTransaction) - .getBytes(StandardCharsets.UTF_8); + var importJob = importProvider.lookup(batchImportSlug).get(); + List storageTokens = new ArrayList<>(); - String storageToken = storageService.store(serialized); + importerProviders.stream() + .filter(provider -> provider.supports(importJobSettings.importConfiguration())) + .findFirst() + .ifPresentOrElse( + provider -> provider.readTransactions( + transactionDTO -> { + var serialized = mapper.writeSafe(transactionDTO) + .getBytes(StandardCharsets.UTF_8); + storageTokens.add(storageService.store(serialized)); + }, + importJobSettings.importConfiguration(), + importJob), + () -> log.warn("No importer provider found for configuration: {}", importJobSettings.importConfiguration()) + ); - var storageTokens = (ArrayList) execution.getVariableLocal("storageTokens"); - storageTokens.add(storageToken); + execution.setVariableLocal("generateAccounts", importJobSettings.generateAccounts()); + execution.setVariableLocal("applyRules", importJobSettings.applyRules()); + execution.setVariableLocal("targetAccountId", importJobSettings.accountId()); + execution.setVariableLocal("storageTokens", storageTokens); } - - @Override - protected void afterProcess(DelegateExecution execution) { - // no specific implementation required - } - } diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportConfigJson.java b/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportConfigJson.java deleted file mode 100644 index 699f054b..00000000 --- a/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportConfigJson.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.jongsoft.finance.serialized; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.jongsoft.finance.ProcessVariable; -import io.micronaut.serde.annotation.Serdeable; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -import java.io.Serializable; -import java.util.List; - -@Getter -@Setter -@Serdeable -@ToString(of = {"accountId", "headers", "dateFormat", "delimiter"}) -public class ImportConfigJson implements ProcessVariable { - - public enum MappingRole { - IGNORE("_ignore"), - DATE("transaction-date"), - BOOK_DATE("booking-date"), - INTEREST_DATE("interest-date"), - OPPOSING_NAME("opposing-name"), - OPPOSING_IBAN("opposing-iban"), - ACCOUNT_IBAN("account-iban"), - AMOUNT("amount"), - CUSTOM_INDICATOR("custom-indicator"), - DESCRIPTION("description"); - - private final String label; - - MappingRole(String label) { - this.label = label; - } - - public String getLabel() { - return label; - } - - public static MappingRole value(String source) { - for (MappingRole role : values()) { - if (role.label.equalsIgnoreCase(source)) { - return role; - } - } - throw new IllegalStateException("No mapping role found for " + source); - } - } - - @JsonProperty("has-headers") - private boolean headers; - - @JsonProperty("apply-rules") - private boolean applyRules; - - @JsonProperty("generate-accounts") - private boolean generateAccounts; - - @JsonProperty("date-format") - private String dateFormat; - - @JsonProperty("delimiter") - private Character delimiter; - - @JsonProperty("column-roles") - private List columnRoles; - - @JsonProperty("custom-indicator") - private CustomIndicator customIndicator; - - private Long accountId; - - @Getter - @Setter - @Serdeable - public static class CustomIndicator implements Serializable { - @JsonProperty("deposit") - private String deposit; - - @JsonProperty("credit") - private String credit; - } - -} diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportJobSettings.java b/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportJobSettings.java new file mode 100644 index 00000000..59ad963a --- /dev/null +++ b/bpmn-process/src/main/java/com/jongsoft/finance/serialized/ImportJobSettings.java @@ -0,0 +1,17 @@ +package com.jongsoft.finance.serialized; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.jongsoft.finance.ProcessVariable; +import com.jongsoft.finance.importer.api.ImporterConfiguration; +import io.micronaut.serde.annotation.Serdeable; + +@Serdeable +public record ImportJobSettings( + @JsonProperty + ImporterConfiguration importConfiguration, + @JsonProperty + boolean applyRules, + @JsonProperty + boolean generateAccounts, + @JsonProperty + Long accountId) implements ProcessVariable {} diff --git a/bpmn-process/src/main/java/com/jongsoft/finance/serialized/serde/MappingRoleSerde.java b/bpmn-process/src/main/java/com/jongsoft/finance/serialized/serde/MappingRoleSerde.java deleted file mode 100644 index fe62aec7..00000000 --- a/bpmn-process/src/main/java/com/jongsoft/finance/serialized/serde/MappingRoleSerde.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jongsoft.finance.serialized.serde; - -import com.jongsoft.finance.serialized.ImportConfigJson; -import io.micronaut.core.annotation.NonNull; -import io.micronaut.core.annotation.Nullable; -import io.micronaut.core.type.Argument; -import io.micronaut.serde.Decoder; -import io.micronaut.serde.Encoder; -import io.micronaut.serde.Serde; -import jakarta.inject.Singleton; - -import java.io.IOException; - -@Singleton -public class MappingRoleSerde implements Serde { - @Override - public @Nullable ImportConfigJson.MappingRole deserialize( - @NonNull Decoder decoder, - @NonNull DecoderContext context, - @NonNull Argument type) throws IOException { - return ImportConfigJson.MappingRole.value(decoder.decodeString()); - } - - @Override - public void serialize( - @NonNull Encoder encoder, - @NonNull EncoderContext context, - @NonNull Argument type, ImportConfigJson.@NonNull MappingRole value) throws IOException { - encoder.encodeString(value.getLabel()); - } -} diff --git a/bpmn-process/src/main/resources/bpmn/transaction/import-job.bpmn b/bpmn-process/src/main/resources/bpmn/transaction/import-job.bpmn index af4f6ede..d245aa17 100644 --- a/bpmn-process/src/main/resources/bpmn/transaction/import-job.bpmn +++ b/bpmn-process/src/main/resources/bpmn/transaction/import-job.bpmn @@ -38,7 +38,7 @@ When calling the flow the following variables must be set: - ${importConfiguration.getAccountId()} + ${importConfiguration.accountId()} ${id} @@ -60,9 +60,6 @@ When calling the flow the following variables must be set: ${importJobSlug} ${importConfiguration} - storageTokens - generateAccounts - applyRules ${storageTokens} ${generateAccounts} ${applyRules} @@ -131,7 +128,6 @@ When calling the flow the following variables must be set: ${account_extract.name} ${account_extract.description} - accountId ${accountId} @@ -169,8 +165,6 @@ When calling the flow the following variables must be set: ${importJobSlug} ${importConfiguration} - extractionResult - ${extractionResult} ${locatable} @@ -247,7 +241,6 @@ When calling the flow the following variables must be set: ${transactionToken} - transaction ${transaction} diff --git a/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ApplicationContext.java b/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ApplicationContext.java index 0c80eb1c..1fc44e5d 100644 --- a/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ApplicationContext.java +++ b/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ApplicationContext.java @@ -142,8 +142,8 @@ ImportProvider importProvider() { } @Singleton - CSVConfigProvider csvConfigProvider() { - return Mockito.mock(CSVConfigProvider.class); + ImportConfigurationProvider csvConfigProvider() { + return Mockito.mock(ImportConfigurationProvider.class); } @Singleton diff --git a/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ImportJobIT.java b/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ImportJobIT.java index 550b258f..209b5857 100644 --- a/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ImportJobIT.java +++ b/bpmn-process/src/test/java/com/jongsoft/finance/bpmn/ImportJobIT.java @@ -8,7 +8,7 @@ import com.jongsoft.finance.domain.importer.BatchImportConfig; import com.jongsoft.finance.domain.transaction.Transaction; import com.jongsoft.finance.providers.AccountProvider; -import com.jongsoft.finance.serialized.ImportConfigJson; +import com.jongsoft.finance.serialized.ImportJobSettings; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.DisplayName; @@ -42,10 +42,14 @@ void runImportAccountNotFound(RuntimeContext context) { process .task("task_configure") - .updateVariable("initialConfig", "updatedConfig", config -> { - config.setAccountId(TARGET_ACCOUNT_ID); - return config; - }) + .updateVariable( + "initialConfig", + "updatedConfig", + config -> new ImportJobSettings( + config.importConfiguration(), + config.applyRules(), + config.generateAccounts(), + TARGET_ACCOUNT_ID)) .complete(); // Task should be active again @@ -70,10 +74,14 @@ void runWithAccountMappingAdjustments(RuntimeContext context) { )); process.task("task_configure") - .updateVariable("initialConfig", "updatedConfig", config -> { - config.setAccountId(TARGET_ACCOUNT_ID); - return config; - }) + .updateVariable( + "initialConfig", + "updatedConfig", + config -> new ImportJobSettings( + config.importConfiguration(), + config.applyRules(), + config.generateAccounts(), + TARGET_ACCOUNT_ID)) .complete(); process.task("confirm_mappings") @@ -118,11 +126,14 @@ void runWithManualAccountCreate(RuntimeContext context) { )); process.task("task_configure") - .updateVariable("initialConfig", "updatedConfig", config -> { - config.setAccountId(TARGET_ACCOUNT_ID); - config.setGenerateAccounts(false); - return config; - }) + .updateVariable( + "initialConfig", + "updatedConfig", + config -> new ImportJobSettings( + config.importConfiguration(), + config.applyRules(), + false, + TARGET_ACCOUNT_ID)) .complete(); process.task("confirm_mappings") @@ -165,11 +176,14 @@ void runWithAutomatedAccountCreation(RuntimeContext context) { )); process.task("task_configure") - .updateVariable("initialConfig", "updatedConfig", config -> { - config.setAccountId(TARGET_ACCOUNT_ID); - config.setGenerateAccounts(true); - return config; - }) + .updateVariable( + "initialConfig", + "updatedConfig", + config -> new ImportJobSettings( + config.importConfiguration(), + config.applyRules(), + true, + TARGET_ACCOUNT_ID)) .complete(); process.task("confirm_mappings") @@ -269,6 +283,7 @@ private BatchImport createBatchImport() { .created(new Date()) .fileCode(CSV_FILE_CODE) .config(BatchImportConfig.builder() + .type("CSVImportProvider") .fileCode(JSON_FILE_CODE) .build()) .build(); diff --git a/bpmn-process/src/test/resources/import-test/import-test.csv b/bpmn-process/src/test/resources/import-test/import-test.csv index 5a31c134..bcb4a999 100644 --- a/bpmn-process/src/test/resources/import-test/import-test.csv +++ b/bpmn-process/src/test/resources/import-test/import-test.csv @@ -1,5 +1,5 @@ -Datum,Naam/Omschrijving,Rekening,Tegenrekening,Code,Af/Bij,Bedrag,Valuta,Mutatiesoort,Betalingskenmerk,Mededeling -20160531,Janssen PA,NL20INGB0001234567,NL69INGB0123454789,GT,Bij,"14,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399,, -20160531,MW GA Pieterse,NL20INGB0001234567,NL71INGB0009876543,GT,Af,"283,90",Internetbankieren,Naam: Mw G A Pieterse Omschrijving: inzake bestelling IBAN: NL71INGB0009876543,, -20160531,P. Post,NL20INGB0001234567,NL69INGB0123456789,GT,Bij,"14,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399,, -20160525,Janssen PA,NL20INGB0001234567,NL69INGB0123454789,GT,Bij,"12,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399,, +Datum,Naam/Omschrijving,Rekening,Tegenrekening,Code,Af/Bij,Bedrag,Valuta,Mutatiesoort +20160531,Janssen PA,NL20INGB0001234567,NL69INGB0123454789,GT,Bij,"14,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399 +20160531,MW GA Pieterse,NL20INGB0001234567,NL71INGB0009876543,GT,Af,"283,90",Internetbankieren,Naam: Mw G A Pieterse Omschrijving: inzake bestelling IBAN: NL71INGB0009876543 +20160531,P. Post,NL20INGB0001234567,NL69INGB0123456789,GT,Bij,"14,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399 +20160525,Janssen PA,NL20INGB0001234567,NL69INGB0123454789,GT,Bij,"12,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399 diff --git a/bpmn-process/src/test/resources/logback.xml b/bpmn-process/src/test/resources/logback.xml index 423f1e4f..3ef0b7ce 100644 --- a/bpmn-process/src/test/resources/logback.xml +++ b/bpmn-process/src/test/resources/logback.xml @@ -9,7 +9,7 @@ - + diff --git a/domain/src/main/java/com/jongsoft/finance/domain/importer/BatchImportConfig.java b/domain/src/main/java/com/jongsoft/finance/domain/importer/BatchImportConfig.java index 04335821..dc520be5 100644 --- a/domain/src/main/java/com/jongsoft/finance/domain/importer/BatchImportConfig.java +++ b/domain/src/main/java/com/jongsoft/finance/domain/importer/BatchImportConfig.java @@ -22,21 +22,23 @@ public class BatchImportConfig implements AggregateBase, Serializable { private String name; private String fileCode; + private String type; private transient UserAccount user; @BusinessMethod - public BatchImportConfig(UserAccount user, String name, String fileCode) { + public BatchImportConfig(UserAccount user, String type, String name, String fileCode) { this.user = user; this.name = name; this.fileCode = fileCode; + this.type = type; EventBus.getBus().send( - new CreateConfigurationCommand(name, fileCode)); + new CreateConfigurationCommand(type, name, fileCode)); } - public BatchImport createImport(String content) { - return new BatchImport(this, this.user, content); + public BatchImport createImport(String fileCode) { + return new BatchImport(this, this.user, fileCode); } } diff --git a/domain/src/main/java/com/jongsoft/finance/domain/user/UserAccount.java b/domain/src/main/java/com/jongsoft/finance/domain/user/UserAccount.java index e5294494..16142a5e 100644 --- a/domain/src/main/java/com/jongsoft/finance/domain/user/UserAccount.java +++ b/domain/src/main/java/com/jongsoft/finance/domain/user/UserAccount.java @@ -182,12 +182,12 @@ public TransactionRule createRule(String name, boolean restrictive) { * @return the newly created configuration */ @BusinessMethod - public BatchImportConfig createImportConfiguration(String name, String fileCode) { + public BatchImportConfig createImportConfiguration(String type, String name, String fileCode) { if (notFullUser()) { throw StatusException.notAuthorized("User cannot create import configuration, incorrect privileges."); } - return new BatchImportConfig(this, name, fileCode); + return new BatchImportConfig(this, type, name, fileCode); } /** diff --git a/domain/src/main/java/com/jongsoft/finance/messaging/commands/importer/CreateConfigurationCommand.java b/domain/src/main/java/com/jongsoft/finance/messaging/commands/importer/CreateConfigurationCommand.java index f4450a70..a1787d6c 100644 --- a/domain/src/main/java/com/jongsoft/finance/messaging/commands/importer/CreateConfigurationCommand.java +++ b/domain/src/main/java/com/jongsoft/finance/messaging/commands/importer/CreateConfigurationCommand.java @@ -2,5 +2,5 @@ import com.jongsoft.finance.core.ApplicationEvent; -public record CreateConfigurationCommand(String name, String fileCode) implements ApplicationEvent { +public record CreateConfigurationCommand(String type, String name, String fileCode) implements ApplicationEvent { } diff --git a/domain/src/main/java/com/jongsoft/finance/providers/CSVConfigProvider.java b/domain/src/main/java/com/jongsoft/finance/providers/ImportConfigurationProvider.java similarity index 86% rename from domain/src/main/java/com/jongsoft/finance/providers/CSVConfigProvider.java rename to domain/src/main/java/com/jongsoft/finance/providers/ImportConfigurationProvider.java index f74e51ba..2c8397f3 100644 --- a/domain/src/main/java/com/jongsoft/finance/providers/CSVConfigProvider.java +++ b/domain/src/main/java/com/jongsoft/finance/providers/ImportConfigurationProvider.java @@ -4,7 +4,7 @@ import com.jongsoft.lang.collection.Sequence; import com.jongsoft.lang.control.Optional; -public interface CSVConfigProvider { +public interface ImportConfigurationProvider { Optional lookup(String name); Sequence lookup(); diff --git a/domain/src/test/java/com/jongsoft/finance/domain/user/UserAccountTest.java b/domain/src/test/java/com/jongsoft/finance/domain/user/UserAccountTest.java index 0c1db5d1..4917054b 100644 --- a/domain/src/test/java/com/jongsoft/finance/domain/user/UserAccountTest.java +++ b/domain/src/test/java/com/jongsoft/finance/domain/user/UserAccountTest.java @@ -118,7 +118,7 @@ void createCategory_NotAllowed() { @Test void createBatchConfiguration() { - final BatchImportConfig configuration = fullAccount.createImportConfiguration("test-config", "file-code-sample"); + final BatchImportConfig configuration = fullAccount.createImportConfiguration("CSV", "test-config", "file-code-sample"); assertThat(configuration.getName()).isEqualTo("test-config"); assertThat(configuration.getFileCode()).isEqualTo("file-code-sample"); @@ -128,7 +128,7 @@ void createBatchConfiguration() { @Test void createBatchConfiguration_NotAllowed() { var thrown = assertThrows(StatusException.class, - () -> readOnlyAccount.createImportConfiguration("test-config", "file-code-sample")); + () -> readOnlyAccount.createImportConfiguration("CSV", "test-config", "file-code-sample")); assertThat(thrown.getMessage()).isEqualTo("User cannot create import configuration, incorrect privileges."); } diff --git a/fintrack-api/build.gradle.kts b/fintrack-api/build.gradle.kts index e9593049..da7be999 100644 --- a/fintrack-api/build.gradle.kts +++ b/fintrack-api/build.gradle.kts @@ -44,6 +44,8 @@ dependencies { implementation(project(":jpa-repository")) implementation(project(":rule-engine")) implementation(project(":bpmn-process")) + implementation(project(":transaction-importer:transaction-importer-api")) + implementation(project(":transaction-importer:transaction-importer-csv")) // needed for application.yml runtimeOnly(mn.snakeyaml) diff --git a/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/BatchImportResource.java b/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/BatchImportResource.java index 23cffdb2..d3534296 100644 --- a/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/BatchImportResource.java +++ b/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/BatchImportResource.java @@ -1,7 +1,7 @@ package com.jongsoft.finance.rest.importer; import com.jongsoft.finance.core.exception.StatusException; -import com.jongsoft.finance.providers.CSVConfigProvider; +import com.jongsoft.finance.providers.ImportConfigurationProvider; import com.jongsoft.finance.providers.ImportProvider; import com.jongsoft.finance.providers.SettingProvider; import com.jongsoft.finance.rest.model.CSVImporterConfigResponse; @@ -26,13 +26,13 @@ public class BatchImportResource { private final CurrentUserProvider currentUserProvider; - private final CSVConfigProvider csvConfigProvider; + private final ImportConfigurationProvider csvConfigProvider; private final ImportProvider importProvider; private final SettingProvider settingProvider; public BatchImportResource( CurrentUserProvider currentUserProvider, - CSVConfigProvider csvConfigProvider, + ImportConfigurationProvider csvConfigProvider, ImportProvider importProvider, SettingProvider settingProvider) { this.currentUserProvider = currentUserProvider; @@ -125,6 +125,7 @@ CSVImporterConfigResponse createConfig(@Valid @Body CSVImporterConfigCreateReque } return new CSVImporterConfigResponse(currentUserProvider.currentUser().createImportConfiguration( + request.type(), request.name(), request.fileCode())); } diff --git a/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/CSVImporterConfigCreateRequest.java b/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/CSVImporterConfigCreateRequest.java index c8469c6f..7efc7b2d 100644 --- a/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/CSVImporterConfigCreateRequest.java +++ b/fintrack-api/src/main/java/com/jongsoft/finance/rest/importer/CSVImporterConfigCreateRequest.java @@ -1,12 +1,20 @@ package com.jongsoft.finance.rest.importer; import io.micronaut.serde.annotation.Serdeable; +import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotBlank; @Serdeable.Deserializable record CSVImporterConfigCreateRequest( + @NotBlank + @Schema(description = "The type of importer that is to be used") + + String type, + @Schema(description = "The name of the configuration") + @NotBlank String name, + @Schema(description = "The file code to get the contents of the configuration") @NotBlank String fileCode) { } diff --git a/fintrack-api/src/main/java/com/jongsoft/finance/rest/model/CSVImporterConfigResponse.java b/fintrack-api/src/main/java/com/jongsoft/finance/rest/model/CSVImporterConfigResponse.java index 41b308b1..8512c707 100644 --- a/fintrack-api/src/main/java/com/jongsoft/finance/rest/model/CSVImporterConfigResponse.java +++ b/fintrack-api/src/main/java/com/jongsoft/finance/rest/model/CSVImporterConfigResponse.java @@ -13,16 +13,21 @@ public CSVImporterConfigResponse(BatchImportConfig wrapped) { this.wrapped = wrapped; } - @Schema(description = "The CSV configuration identifier") + @Schema(description = "The configuration identifier") public Long getId() { return wrapped.getId(); } - @Schema(description = "The name of the CSV configuration") + @Schema(description = "The name of the configuration") public String getName() { return wrapped.getName(); } + @Schema(description = "The type of importer that will be used") + public String getType() { + return wrapped.getType(); + } + @Schema(description = "The file code to get the contents of the configuration") public String getFile() { return wrapped.getFileCode(); diff --git a/fintrack-api/src/main/java/com/jongsoft/finance/rest/process/VariableMap.java b/fintrack-api/src/main/java/com/jongsoft/finance/rest/process/VariableMap.java index acd9cf4c..cc528dbb 100644 --- a/fintrack-api/src/main/java/com/jongsoft/finance/rest/process/VariableMap.java +++ b/fintrack-api/src/main/java/com/jongsoft/finance/rest/process/VariableMap.java @@ -1,5 +1,6 @@ package com.jongsoft.finance.rest.process; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.jongsoft.finance.ProcessVariable; import io.micronaut.serde.annotation.Serdeable; import io.swagger.v3.oas.annotations.media.Schema; @@ -19,7 +20,9 @@ public record VariableList(List content) implements ProcessVari @Serdeable @Schema(name = "WrappedVariable", description = "A variable wrapped for the task.") - public record WrappedVariable(T value) implements ProcessVariable { + public record WrappedVariable( + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "_type") + T value) implements ProcessVariable { } @Schema(description = "The actual map of all the variables set for the task.") diff --git a/fintrack-api/src/main/resources/docs/index.html b/fintrack-api/src/main/resources/docs/index.html index 010e18d1..425bd883 100644 --- a/fintrack-api/src/main/resources/docs/index.html +++ b/fintrack-api/src/main/resources/docs/index.html @@ -3,7 +3,7 @@ - FinTrack: REST API Documentation + Pledger.io: REST API Documentation @@ -24,10 +24,10 @@ allow-server-selection="false">
-
FinTrack API
+
Pledger.io API

- FinTrack is a "self hosted" application that helps in keeping track of your personal finances. + Pledger.io is a "self hosted" application that helps in keeping track of your personal finances. It helps you keep track of your income and expenses to allow you to spend less many and save more.

diff --git a/fintrack-api/src/test/java/com/jongsoft/finance/rest/importer/BatchImportResourceTest.java b/fintrack-api/src/test/java/com/jongsoft/finance/rest/importer/BatchImportResourceTest.java index 98d8ff79..6ffee29a 100644 --- a/fintrack-api/src/test/java/com/jongsoft/finance/rest/importer/BatchImportResourceTest.java +++ b/fintrack-api/src/test/java/com/jongsoft/finance/rest/importer/BatchImportResourceTest.java @@ -4,14 +4,13 @@ import com.jongsoft.finance.core.DateUtils; import com.jongsoft.finance.domain.importer.BatchImport; import com.jongsoft.finance.domain.importer.BatchImportConfig; -import com.jongsoft.finance.providers.CSVConfigProvider; +import com.jongsoft.finance.providers.ImportConfigurationProvider; import com.jongsoft.finance.providers.ImportProvider; import com.jongsoft.finance.rest.TestSetup; import com.jongsoft.lang.Collections; import com.jongsoft.lang.Control; import io.micronaut.context.annotation.Replaces; import io.micronaut.test.annotation.MockBean; -import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.restassured.specification.RequestSpecification; import jakarta.inject.Inject; import org.hamcrest.Matchers; @@ -29,12 +28,12 @@ class BatchImportResourceTest extends TestSetup { @Inject private ImportProvider importProvider; @Inject - private CSVConfigProvider csvConfigProvider; + private ImportConfigurationProvider csvConfigProvider; @Replaces @MockBean - CSVConfigProvider configProvider() { - return Mockito.mock(CSVConfigProvider.class); + ImportConfigurationProvider configProvider() { + return Mockito.mock(ImportConfigurationProvider.class); } @Replaces @@ -209,7 +208,10 @@ void createConfig(RequestSpecification spec) { // @formatter:off spec .given() - .body(Map.of("name", "sample-configuration", "fileCode", "token-sample")) + .body(Map.of( + "type", "csv", + "name", "sample-configuration", + "fileCode", "token-sample")) .when() .put("/api/import/config") .then() diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateConfigurationHandler.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateConfigurationHandler.java index 82090e84..bf6257b0 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateConfigurationHandler.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateConfigurationHandler.java @@ -1,7 +1,7 @@ package com.jongsoft.finance.jpa.importer; import com.jongsoft.finance.annotation.BusinessEventListener; -import com.jongsoft.finance.jpa.importer.entity.CSVImportConfig; +import com.jongsoft.finance.jpa.importer.entity.ImportConfig; import com.jongsoft.finance.jpa.reactive.ReactiveEntityManager; import com.jongsoft.finance.jpa.user.entity.UserAccountJpa; import com.jongsoft.finance.messaging.CommandHandler; @@ -32,9 +32,10 @@ public CreateConfigurationHandler(ReactiveEntityManager entityManager, Authentic public void handle(CreateConfigurationCommand command) { log.info("[{}] - Processing CSV configuration create event", command.name()); - var entity = CSVImportConfig.builder() + var entity = ImportConfig.builder() .fileCode(command.fileCode()) .name(command.name()) + .type(command.type()) .user(entityManager.get(UserAccountJpa.class, Collections.Map("username", authenticationFacade.authenticated()))) .build(); diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateImportJobHandler.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateImportJobHandler.java index 42ae7cf6..df8f781f 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateImportJobHandler.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CreateImportJobHandler.java @@ -2,7 +2,7 @@ import com.jongsoft.finance.annotation.BusinessEventListener; import com.jongsoft.finance.core.exception.StatusException; -import com.jongsoft.finance.jpa.importer.entity.CSVImportConfig; +import com.jongsoft.finance.jpa.importer.entity.ImportConfig; import com.jongsoft.finance.jpa.importer.entity.ImportJpa; import com.jongsoft.finance.jpa.reactive.ReactiveEntityManager; import com.jongsoft.finance.messaging.CommandHandler; @@ -29,8 +29,8 @@ public CreateImportJobHandler(ReactiveEntityManager entityManager) { public void handle(CreateImportJobCommand command) { log.info("[{}] - Processing import create event", command.slug()); - var configJpa = entityManager.blocking() - .hql("from CSVImportConfig where id = :id") + var configJpa = entityManager.blocking() + .hql("from ImportConfig where id = :id") .set("id", command.configId()) .maybe() .getOrThrow(() -> diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpa.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportConfigurationProviderJpa.java similarity index 73% rename from jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpa.java rename to jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportConfigurationProviderJpa.java index 90421ffa..80646dbb 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpa.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportConfigurationProviderJpa.java @@ -2,9 +2,9 @@ import com.jongsoft.finance.domain.importer.BatchImportConfig; import com.jongsoft.finance.domain.user.UserAccount; -import com.jongsoft.finance.jpa.importer.entity.CSVImportConfig; +import com.jongsoft.finance.jpa.importer.entity.ImportConfig; import com.jongsoft.finance.jpa.reactive.ReactiveEntityManager; -import com.jongsoft.finance.providers.CSVConfigProvider; +import com.jongsoft.finance.providers.ImportConfigurationProvider; import com.jongsoft.finance.security.AuthenticationFacade; import com.jongsoft.lang.collection.Sequence; import com.jongsoft.lang.control.Optional; @@ -15,27 +15,27 @@ @Slf4j @ReadOnly @Singleton -public class CSVConfigProviderJpa implements CSVConfigProvider { +public class ImportConfigurationProviderJpa implements ImportConfigurationProvider { private final ReactiveEntityManager entityManager; private final AuthenticationFacade authenticationFacade; - public CSVConfigProviderJpa(ReactiveEntityManager entityManager, AuthenticationFacade authenticationFacade) { + public ImportConfigurationProviderJpa(ReactiveEntityManager entityManager, AuthenticationFacade authenticationFacade) { this.entityManager = entityManager; this.authenticationFacade = authenticationFacade; } @Override public Optional lookup(String name) { - log.trace("CSVConfiguration lookup by name {}", name); + log.trace("Import configuration lookup by name {}", name); var hql = """ - select b from CSVImportConfig b + select b from ImportConfig b where b.name = :name and b.user.username = :username """; - return entityManager.blocking() + return entityManager.blocking() .hql(hql) .set("name", name) .set("username", authenticationFacade.authenticated()) @@ -48,17 +48,17 @@ public Sequence lookup() { log.trace("CSVConfiguration listing"); var hql = """ - select b from CSVImportConfig b + select b from ImportConfig b where b.user.username = :username"""; - return entityManager.blocking() + return entityManager.blocking() .hql(hql) .set("username", authenticationFacade.authenticated()) .sequence() .map(this::convert); } - private BatchImportConfig convert(CSVImportConfig source) { + private BatchImportConfig convert(ImportConfig source) { if (source == null) { return null; } @@ -66,6 +66,7 @@ private BatchImportConfig convert(CSVImportConfig source) { return BatchImportConfig.builder() .id(source.getId()) .name(source.getName()) + .type(source.getType()) .fileCode(source.getFileCode()) .user(UserAccount.builder() .id(source.getUser().getId()) diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportProviderJpa.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportProviderJpa.java index 46c768b0..a696018f 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportProviderJpa.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/ImportProviderJpa.java @@ -72,6 +72,7 @@ protected BatchImport convert(ImportJpa source) { .slug(source.getSlug()) .finished(source.getFinished()) .config(BatchImportConfig.builder() + .type(source.getConfig().getType()) .name(source.getConfig().getName()) .fileCode(source.getConfig().getFileCode()) .build()) diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/CSVImportConfig.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportConfig.java similarity index 74% rename from jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/CSVImportConfig.java rename to jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportConfig.java index 9a9178fd..ea9a13dc 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/CSVImportConfig.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportConfig.java @@ -15,24 +15,28 @@ @Entity @Getter @Table(name = "import_config") -public class CSVImportConfig extends EntityJpa { +public class ImportConfig extends EntityJpa { private String name; @Column private String fileCode; + @Column + private String type; + @ManyToOne @JoinColumn private UserAccountJpa user; @Builder - private CSVImportConfig(String name, String fileCode, UserAccountJpa user) { + private ImportConfig(String name, String fileCode, String type, UserAccountJpa user) { this.name = name; this.fileCode = fileCode; this.user = user; + this.type = type; } - protected CSVImportConfig() { + protected ImportConfig() { } } diff --git a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportJpa.java b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportJpa.java index 8ee6488f..74b13083 100644 --- a/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportJpa.java +++ b/jpa-repository/src/main/java/com/jongsoft/finance/jpa/importer/entity/ImportJpa.java @@ -25,7 +25,7 @@ public class ImportJpa extends EntityJpa { @ManyToOne @JoinColumn - private CSVImportConfig config; + private ImportConfig config; @ManyToOne @JoinColumn @@ -40,7 +40,7 @@ private ImportJpa( Date finished, String slug, String fileCode, - CSVImportConfig config, + ImportConfig config, UserAccountJpa user, boolean archived, List transactions) { diff --git a/jpa-repository/src/main/resources/db/migration/V20240404131810__add_import_type.sql b/jpa-repository/src/main/resources/db/migration/V20240404131810__add_import_type.sql new file mode 100644 index 00000000..85286b2a --- /dev/null +++ b/jpa-repository/src/main/resources/db/migration/V20240404131810__add_import_type.sql @@ -0,0 +1,5 @@ +-- add the type column and default all existing to CSVImportProvider + +ALTER TABLE import_config + ADD COLUMN type VARCHAR(255) DEFAULT 'CSVImportProvider' NOT NULL; + diff --git a/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigEventListenerIT.java b/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigEventListenerIT.java index 4e5e358a..a61f1593 100644 --- a/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigEventListenerIT.java +++ b/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigEventListenerIT.java @@ -1,7 +1,7 @@ package com.jongsoft.finance.jpa.importer; import com.jongsoft.finance.jpa.JpaTestSetup; -import com.jongsoft.finance.jpa.importer.entity.CSVImportConfig; +import com.jongsoft.finance.jpa.importer.entity.ImportConfig; import com.jongsoft.finance.messaging.commands.importer.CreateConfigurationCommand; import com.jongsoft.finance.security.AuthenticationFacade; import io.micronaut.context.event.ApplicationEventPublisher; @@ -38,13 +38,14 @@ void setup() { @Test void handleCreatedEvent() { eventPublisher.publishEvent( - new CreateConfigurationCommand("test-config", "file-code-3")); + new CreateConfigurationCommand("CSVImportProvider", "test-config", "file-code-3")); - var query = entityManager.createQuery("select c from CSVImportConfig c where c.name = 'test-config'"); - var check = (CSVImportConfig) query.getSingleResult(); + var query = entityManager.createQuery("select c from ImportConfig c where c.name = 'test-config'"); + var check = (ImportConfig) query.getSingleResult(); Assertions.assertThat(check.getName()).isEqualTo("test-config"); Assertions.assertThat(check.getFileCode()).isEqualTo("file-code-3"); + Assertions.assertThat(check.getType()).isEqualTo("CSVImportProvider"); } @MockBean diff --git a/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpaIT.java b/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpaIT.java index b42dcdd0..866429b0 100644 --- a/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpaIT.java +++ b/jpa-repository/src/test/java/com/jongsoft/finance/jpa/importer/CSVConfigProviderJpaIT.java @@ -1,7 +1,7 @@ package com.jongsoft.finance.jpa.importer; import com.jongsoft.finance.jpa.JpaTestSetup; -import com.jongsoft.finance.providers.CSVConfigProvider; +import com.jongsoft.finance.providers.ImportConfigurationProvider; import com.jongsoft.finance.security.AuthenticationFacade; import io.micronaut.test.annotation.MockBean; import jakarta.inject.Inject; @@ -16,7 +16,7 @@ class CSVConfigProviderJpaIT extends JpaTestSetup { private AuthenticationFacade authenticationFacade; @Inject - private CSVConfigProvider csvConfigProvider; + private ImportConfigurationProvider csvConfigProvider; @BeforeEach void setup() { @@ -33,7 +33,10 @@ void lookup() { Assertions.assertThat(csvConfigProvider.lookup()) .hasSize(1) .first() - .satisfies(batch -> Assertions.assertThat(batch.getFileCode()).isEqualTo("file-code-1")); + .satisfies(batch -> { + Assertions.assertThat(batch.getFileCode()).isEqualTo("file-code-1"); + Assertions.assertThat(batch.getType()).isEqualTo("CSVImportProvider"); + }); } @Test diff --git a/rule-engine/build.gradle.kts b/rule-engine/build.gradle.kts index 6c5bc1e5..c5bc9c6a 100644 --- a/rule-engine/build.gradle.kts +++ b/rule-engine/build.gradle.kts @@ -1,11 +1,11 @@ -plugins { - id("io.freefair.lombok") -} - dependencies { + annotationProcessor(mn.lombok) + implementation(libs.lang) implementation(project(":core")) implementation(project(":domain")) + compileOnly(mn.lombok) + testImplementation(libs.bundles.junit) } diff --git a/rule-engine/src/main/java/com/jongsoft/finance/rule/impl/RuleEngineImpl.java b/rule-engine/src/main/java/com/jongsoft/finance/rule/impl/RuleEngineImpl.java index c505766a..1f8669e7 100644 --- a/rule-engine/src/main/java/com/jongsoft/finance/rule/impl/RuleEngineImpl.java +++ b/rule-engine/src/main/java/com/jongsoft/finance/rule/impl/RuleEngineImpl.java @@ -16,12 +16,17 @@ import java.util.List; @Singleton -@RequiredArgsConstructor(onConstructor_ = @Inject) public class RuleEngineImpl implements RuleEngine { private final TransactionRuleProvider transactionRuleProvider; private final List locators; + @Inject + public RuleEngineImpl(TransactionRuleProvider transactionRuleProvider, List locators) { + this.transactionRuleProvider = transactionRuleProvider; + this.locators = locators; + } + @Override public RuleDataSet run(RuleDataSet input) { var outputSet = new RuleDataSet(); diff --git a/settings.gradle.kts b/settings.gradle.kts index e4876ce3..24d1a72e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -45,4 +45,12 @@ dependencyResolutionManagement { } } -include("core", "domain", "rule-engine", "bpmn-process", "jpa-repository", "fintrack-api") +include( + "core", + "domain", + "transaction-importer:transaction-importer-api", + "transaction-importer:transaction-importer-csv", + "rule-engine", + "bpmn-process", + "jpa-repository", + "fintrack-api") diff --git a/transaction-importer/transaction-importer-api/build.gradle.kts b/transaction-importer/transaction-importer-api/build.gradle.kts new file mode 100644 index 00000000..c19689c0 --- /dev/null +++ b/transaction-importer/transaction-importer-api/build.gradle.kts @@ -0,0 +1,5 @@ +dependencies { + implementation(mn.micronaut.serde.jackson) + implementation(project(":core")) + implementation(project(":domain")) +} \ No newline at end of file diff --git a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImporterProvider.java b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImporterProvider.java new file mode 100644 index 00000000..43bbecdf --- /dev/null +++ b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/ImporterProvider.java @@ -0,0 +1,20 @@ +package com.jongsoft.finance.importer; + +import com.jongsoft.finance.domain.importer.BatchImport; +import com.jongsoft.finance.domain.importer.BatchImportConfig; +import com.jongsoft.finance.importer.api.ImporterConfiguration; +import com.jongsoft.finance.importer.api.TransactionConsumer; + +public interface ImporterProvider { + + void readTransactions(TransactionConsumer consumer, ImporterConfiguration updatedConfiguration, BatchImport importJob); + + T loadConfiguration(BatchImportConfig batchImportConfig); + + boolean supports(X configuration); + + default String getImporterType() { + return this.getClass().getSimpleName(); + } + +} diff --git a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/ImporterConfiguration.java b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/ImporterConfiguration.java new file mode 100644 index 00000000..d4dcf160 --- /dev/null +++ b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/ImporterConfiguration.java @@ -0,0 +1,7 @@ +package com.jongsoft.finance.importer.api; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "type") +public interface ImporterConfiguration { +} diff --git a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionConsumer.java b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionConsumer.java new file mode 100644 index 00000000..a42f4c15 --- /dev/null +++ b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionConsumer.java @@ -0,0 +1,6 @@ +package com.jongsoft.finance.importer.api; + +import java.util.function.Consumer; + +public interface TransactionConsumer extends Consumer { +} diff --git a/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionDTO.java b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionDTO.java new file mode 100644 index 00000000..c99a2ab8 --- /dev/null +++ b/transaction-importer/transaction-importer-api/src/main/java/com/jongsoft/finance/importer/api/TransactionDTO.java @@ -0,0 +1,31 @@ +package com.jongsoft.finance.importer.api; + +import com.jongsoft.finance.core.TransactionType; +import io.micronaut.serde.annotation.Serdeable; + +import java.time.LocalDate; + +@Serdeable +public record TransactionDTO( + // The amount of the transaction + double amount, + // The type of the transaction + TransactionType type, + // The description of the transaction + String description, + // The date of the transaction + LocalDate transactionDate, + // The date the transaction starts to accrue interest + LocalDate interestDate, + // The date the transaction was booked + LocalDate bookDate, + // The IBAN of the opposing account + String opposingIBAN, + // The name of the opposing account + String opposingName) { + + @Override + public String toString() { + return "Transfer of " + amount + " to " + opposingName + " (" + opposingIBAN + ") on " + transactionDate; + } +} diff --git a/transaction-importer/transaction-importer-csv/build.gradle.kts b/transaction-importer/transaction-importer-csv/build.gradle.kts new file mode 100644 index 00000000..c60de5ce --- /dev/null +++ b/transaction-importer/transaction-importer-csv/build.gradle.kts @@ -0,0 +1,17 @@ + +micronaut { + testRuntime("junit5") +} + +dependencies { + implementation(libs.csv) + implementation(libs.lang) + implementation(mn.micronaut.serde.jackson) + + implementation(project(":transaction-importer:transaction-importer-api")) + implementation(project(":core")) + implementation(project(":domain")) + + testImplementation(libs.bundles.junit) + testRuntimeOnly(mn.logback.classic) +} \ No newline at end of file diff --git a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVConfiguration.java b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVConfiguration.java new file mode 100644 index 00000000..e58f3f8d --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVConfiguration.java @@ -0,0 +1,21 @@ +package com.jongsoft.finance.importer.csv; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.jongsoft.finance.importer.api.ImporterConfiguration; +import io.micronaut.serde.annotation.Serdeable; + +import java.util.List; + +@Serdeable +public record CSVConfiguration( + @JsonProperty("has-headers") + boolean headers, + @JsonProperty("date-format") + String dateFormat, + @JsonProperty("delimiter") + char delimiter, + @JsonProperty("custom-indicator") + TransactionTypeIndicator transactionTypeIndicator, + @JsonProperty("column-roles") + List columnRoles) implements ImporterConfiguration { +} diff --git a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVImportProvider.java b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVImportProvider.java new file mode 100644 index 00000000..5edae6f7 --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/CSVImportProvider.java @@ -0,0 +1,134 @@ +package com.jongsoft.finance.importer.csv; + +import com.jongsoft.finance.StorageService; +import com.jongsoft.finance.core.TransactionType; +import com.jongsoft.finance.domain.importer.BatchImport; +import com.jongsoft.finance.domain.importer.BatchImportConfig; +import com.jongsoft.finance.importer.ImporterProvider; +import com.jongsoft.finance.importer.api.ImporterConfiguration; +import com.jongsoft.finance.importer.api.TransactionConsumer; +import com.jongsoft.finance.importer.api.TransactionDTO; +import com.jongsoft.lang.Control; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.exceptions.CsvValidationException; +import io.micronaut.serde.ObjectMapper; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.function.Function; + +@Singleton +public class CSVImportProvider implements ImporterProvider { + private final Logger logger = LoggerFactory.getLogger(CSVImportProvider.class); + + private final StorageService storageService; + private final ObjectMapper objectMapper; + + @Inject + public CSVImportProvider(StorageService storageService, ObjectMapper objectMapper) { + this.storageService = storageService; + this.objectMapper = objectMapper; + } + + @Override + public void readTransactions(TransactionConsumer consumer, ImporterConfiguration configuration, BatchImport importJob) { + logger.info("Reading transactions from CSV file: {}", importJob.getSlug()); + var csvConfiguration = (CSVConfiguration) configuration; + + try { + var inputStream = storageService.read(importJob.getFileCode()) + .map(ByteArrayInputStream::new) + .map(InputStreamReader::new) + .getOrThrow(() -> new IllegalStateException("Failed to read CSV file: " + importJob.getFileCode())); + + try (var reader = new CSVReaderBuilder(inputStream) + .withCSVParser(new CSVParserBuilder() + .withSeparator(csvConfiguration.delimiter()) + .build()) + .build()) { + + if (csvConfiguration.headers()) { + logger.debug("CSV file has headers, skipping first line"); + reader.skip(1); + } + + String[] line; + while ((line = reader.readNext()) != null) { + if (line.length != csvConfiguration.columnRoles().size()) { + logger.warn("Skipping line, columns found {} but expected is {}: {}", + line.length, + csvConfiguration.columnRoles().size(), + line); + continue; + } + + consumer.accept(readLine(line, csvConfiguration)); + } + } + } catch (IOException | CsvValidationException e) { + logger.warn("Failed to read CSV file: {}", importJob.getFileCode(), e); + } + } + + @Override + public CSVConfiguration loadConfiguration(BatchImportConfig batchImportConfig) { + logger.debug("Loading CSV configuration from disk: {}", batchImportConfig.getFileCode()); + + try { + var jsonBytes = storageService.read(batchImportConfig.getFileCode()); + if (jsonBytes.isPresent()) { + return objectMapper.readValue(jsonBytes.get(), CSVConfiguration.class); + } + + logger.warn("No CSV configuration found on disk: {}", batchImportConfig.getFileCode()); + throw new IllegalStateException("No CSV configuration found on disk: " + batchImportConfig.getFileCode()); + } catch (IOException e) { + logger.warn("Could not load CSV configuration from disk: {}", batchImportConfig.getFileCode(), e); + throw new IllegalStateException("Failed to load CSV configuration from disk: " + batchImportConfig.getFileCode()); + } + } + + @Override + public boolean supports(X configuration) { + return configuration instanceof CSVConfiguration; + } + + private TransactionDTO readLine(String[] line, CSVConfiguration configuration) { + Function columnLocator = (role) -> Control.Try(() -> line[configuration.columnRoles().indexOf(role)]).recover(x -> null).get(); + Function parseDate = (date) -> date != null ? LocalDate.parse(date, DateTimeFormatter.ofPattern(configuration.dateFormat())) : null; + Function parseAmount = (amount) -> Double.parseDouble(amount.replace(',', '.')); + + var amount = parseAmount.apply(columnLocator.apply(ColumnRole.AMOUNT)); + var type = Control.Option(columnLocator.apply(ColumnRole.CUSTOM_INDICATOR)) + .map(indicator -> { + if (indicator.equalsIgnoreCase(configuration.transactionTypeIndicator().credit())) { + return TransactionType.CREDIT; + } else if (indicator.equalsIgnoreCase(configuration.transactionTypeIndicator().deposit())) { + return TransactionType.DEBIT; + } + + return null; + }) + .getOrSupply(() -> amount >= 0 ? TransactionType.DEBIT : TransactionType.CREDIT); + + logger.trace("Reading single transaction on {}: amount={}, type={}", parseDate.apply(columnLocator.apply(ColumnRole.DATE)), amount, type); + + return new TransactionDTO( + amount, + type, + columnLocator.apply(ColumnRole.DESCRIPTION), + parseDate.apply(columnLocator.apply(ColumnRole.DATE)), + parseDate.apply(columnLocator.apply(ColumnRole.INTEREST_DATE)), + parseDate.apply(columnLocator.apply(ColumnRole.BOOK_DATE)), + columnLocator.apply(ColumnRole.OPPOSING_IBAN), + columnLocator.apply(ColumnRole.OPPOSING_NAME)); + } +} diff --git a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/ColumnRole.java b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/ColumnRole.java new file mode 100644 index 00000000..21b1150e --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/ColumnRole.java @@ -0,0 +1,33 @@ +package com.jongsoft.finance.importer.csv; + +public enum ColumnRole { + IGNORE("_ignore"), + DATE("transaction-date"), + BOOK_DATE("booking-date"), + INTEREST_DATE("interest-date"), + OPPOSING_NAME("opposing-name"), + OPPOSING_IBAN("opposing-iban"), + ACCOUNT_IBAN("account-iban"), + AMOUNT("amount"), + CUSTOM_INDICATOR("custom-indicator"), + DESCRIPTION("description"); + + private final String label; + + ColumnRole(String label) { + this.label = label; + } + + public String getLabel() { + return label; + } + + public static ColumnRole value(String source) { + for (ColumnRole role : values()) { + if (role.label.equalsIgnoreCase(source)) { + return role; + } + } + throw new IllegalStateException("No mapping role found for " + source); + } +} diff --git a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/TransactionTypeIndicator.java b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/TransactionTypeIndicator.java new file mode 100644 index 00000000..ad6e9bf6 --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/TransactionTypeIndicator.java @@ -0,0 +1,12 @@ +package com.jongsoft.finance.importer.csv; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.micronaut.serde.annotation.Serdeable; + +@Serdeable +public record TransactionTypeIndicator( + @JsonProperty("deposit") + String deposit, + @JsonProperty("credit") + String credit) { +} diff --git a/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/serde/ColumnRoleSerde.java b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/serde/ColumnRoleSerde.java new file mode 100644 index 00000000..e8065d02 --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/main/java/com/jongsoft/finance/importer/csv/serde/ColumnRoleSerde.java @@ -0,0 +1,23 @@ +package com.jongsoft.finance.importer.csv.serde; + +import com.jongsoft.finance.importer.csv.ColumnRole; +import io.micronaut.core.type.Argument; +import io.micronaut.serde.Decoder; +import io.micronaut.serde.Encoder; +import io.micronaut.serde.Serde; +import jakarta.inject.Singleton; + +import java.io.IOException; + +@Singleton +class ColumnRoleSerde implements Serde { + @Override + public ColumnRole deserialize(Decoder decoder, DecoderContext context, Argument type) throws IOException { + return ColumnRole.value(decoder.decodeString()); + } + + @Override + public void serialize(Encoder encoder, EncoderContext context, Argument type, ColumnRole value) throws IOException { + encoder.encodeString(value.getLabel()); + } +} diff --git a/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java b/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java new file mode 100644 index 00000000..994c60e6 --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/test/java/com/jongsoft/finance/importer/csv/CSVImportProviderTest.java @@ -0,0 +1,151 @@ +package com.jongsoft.finance.importer.csv; + +import com.jongsoft.finance.StorageService; +import com.jongsoft.finance.core.TransactionType; +import com.jongsoft.finance.domain.importer.BatchImport; +import com.jongsoft.finance.domain.importer.BatchImportConfig; +import com.jongsoft.finance.importer.api.TransactionConsumer; +import com.jongsoft.finance.importer.api.TransactionDTO; +import com.jongsoft.lang.Control; +import io.micronaut.serde.ObjectMapper; +import io.micronaut.test.annotation.MockBean; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import jakarta.inject.Inject; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.List; + + +@MicronautTest +class CSVImportProviderTest { + + @Inject + private CSVImportProvider csvImportProvider; + + @Inject + private ObjectMapper objectMapper; + + @Inject + private StorageService storageService; + + @MockBean + StorageService storageService() { + return Mockito.mock(StorageService.class); + } + + @Test + @DisplayName("Load configuration file is missing") + void loadConfiguration_fileMissing() { + Assertions.assertThatException() + .describedAs("No CSV configuration found on disk: my-secret-files") + .isThrownBy(() -> csvImportProvider.loadConfiguration(createBatchImportConfig())) + .isInstanceOf(IllegalStateException.class) + .withMessage("No CSV configuration found on disk: my-secret-files"); + } + + @Test + @DisplayName("Load configuration file that is valid") + void loadConfiguration() throws IOException { + Mockito.when(storageService.read("my-secret-files")) + .thenReturn(Control.Option( + getClass().getResourceAsStream("/configuration/valid-config.json") + .readAllBytes())); + + var configuration = csvImportProvider.loadConfiguration(createBatchImportConfig()); + + Assertions.assertThat(configuration) + .isNotNull() + .extracting(CSVConfiguration::delimiter, CSVConfiguration::headers, CSVConfiguration::dateFormat, CSVConfiguration::transactionTypeIndicator, CSVConfiguration::columnRoles) + .isEqualTo(List.of( + ',', + true, + "yyyyMMdd", + new TransactionTypeIndicator("Bij", "Af"), + List.of( + ColumnRole.DATE, + ColumnRole.OPPOSING_NAME, + ColumnRole.ACCOUNT_IBAN, + ColumnRole.OPPOSING_IBAN, + ColumnRole.IGNORE, + ColumnRole.CUSTOM_INDICATOR, + ColumnRole.AMOUNT, + ColumnRole.IGNORE, + ColumnRole.DESCRIPTION))); + } + + @Test + @DisplayName("Read transactions with single deposit") + void readTransactions_deposit() throws IOException { + Mockito.when(storageService.read("my-secret-import-files")) + .thenReturn(Control.Option( + getClass().getResourceAsStream("/csv-files/single-deposit.csv") + .readAllBytes())); + + var consumer = Mockito.mock(TransactionConsumer.class); + + csvImportProvider.readTransactions(consumer, createCSVConfiguration(), createBatchImport()); + + var transactionCaptor = ArgumentCaptor.forClass(TransactionDTO.class); + Mockito.verify(consumer, Mockito.times(1)).accept(transactionCaptor.capture()); + + var transaction = transactionCaptor.getValue(); + Assertions.assertThat(transaction.amount()).isEqualTo(14.19); + Assertions.assertThat(transaction.opposingIBAN()).isEqualTo("NL69INGB0123454789"); + Assertions.assertThat(transaction.opposingName()).isEqualTo("Janssen PA"); + Assertions.assertThat(transaction.transactionDate()).isEqualTo("2016-05-31"); + Assertions.assertThat(transaction.type()).isEqualTo(TransactionType.DEBIT); + } + + @Test + @DisplayName("Read transactions with single withdrawal") + void readTransactions_withdrawal() throws IOException { + Mockito.when(storageService.read("my-secret-import-files")) + .thenReturn(Control.Option( + getClass().getResourceAsStream("/csv-files/single-withdrawal.csv") + .readAllBytes())); + + var consumer = Mockito.mock(TransactionConsumer.class); + + csvImportProvider.readTransactions(consumer, createCSVConfiguration(), createBatchImport()); + + var transactionCaptor = ArgumentCaptor.forClass(TransactionDTO.class); + Mockito.verify(consumer, Mockito.times(1)).accept(transactionCaptor.capture()); + + var transaction = transactionCaptor.getValue(); + Assertions.assertThat(transaction.amount()).isEqualTo(283.90); + Assertions.assertThat(transaction.opposingIBAN()).isEqualTo("NL71INGB0009876543"); + Assertions.assertThat(transaction.opposingName()).isEqualTo("MW GA Pieterse"); + Assertions.assertThat(transaction.transactionDate()).isEqualTo("2016-05-31"); + Assertions.assertThat(transaction.type()).isEqualTo(TransactionType.CREDIT); + } + + @Test + void exposesCorrectType() { + Assertions.assertThat(csvImportProvider.getImporterType()) + .isEqualTo("CSVImportProvider"); + } + + private BatchImport createBatchImport() { + return BatchImport.builder() + .fileCode("my-secret-import-files") + .slug("this-is-a-import-slug") + .build(); + } + + private CSVConfiguration createCSVConfiguration() throws IOException { + return objectMapper.readValue( + getClass().getResourceAsStream("/configuration/valid-config.json"), + CSVConfiguration.class); + } + + private BatchImportConfig createBatchImportConfig() { + return BatchImportConfig.builder() + .fileCode("my-secret-files") + .build(); + } +} \ No newline at end of file diff --git a/transaction-importer/transaction-importer-csv/src/test/java/org/mockito/configuration/MockitoConfiguration.java b/transaction-importer/transaction-importer-csv/src/test/java/org/mockito/configuration/MockitoConfiguration.java new file mode 100644 index 00000000..65d1702c --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/test/java/org/mockito/configuration/MockitoConfiguration.java @@ -0,0 +1,69 @@ +package org.mockito.configuration; + +import com.jongsoft.finance.ResultPage; +import com.jongsoft.lang.Collections; +import com.jongsoft.lang.Control; +import com.jongsoft.lang.collection.Sequence; +import com.jongsoft.lang.control.Optional; +import org.mockito.configuration.IMockitoConfiguration; +import org.mockito.internal.stubbing.defaultanswers.ReturnsEmptyValues; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** + * This class is used to configure Mockito's default behavior. + * It implements the IMockitoConfiguration interface. + */ +@SuppressWarnings("unused") +public class MockitoConfiguration implements IMockitoConfiguration { + + /** + * This method is used to provide a default answer for all methods of a mock that are not stubbed. + * It overrides the getDefaultAnswer method from the IMockitoConfiguration interface. + * If the return type of the method is assignable from Optional, it returns Control.Option(). + * Otherwise, it uses the super class's answer method to return default values. + * + * @return An Answer that provides the default answer for unstubbed methods. + */ + @Override + public Answer getDefaultAnswer() { + return new ReturnsEmptyValues() { + @Override + public Object answer(InvocationOnMock invocation) { + if (Optional.class.isAssignableFrom(invocation.getMethod().getReturnType())) { + return Control.Option(); + } + if (Sequence.class.isAssignableFrom(invocation.getMethod().getReturnType())) { + return Collections.List(); + } + if (ResultPage.class.isAssignableFrom(invocation.getMethod().getReturnType())) { + return ResultPage.empty(); + } + return super.answer(invocation); + } + + }; + } + + /** + * This method is used to determine whether Mockito should clean the stack trace. + * It overrides the cleansStackTrace method from the IMockitoConfiguration interface. + * + * @return A boolean value indicating whether Mockito should clean the stack trace. It always returns false. + */ + @Override + public boolean cleansStackTrace() { + return false; + } + + /** + * This method is used to determine whether Mockito should enable the class cache. + * It overrides the enableClassCache method from the IMockitoConfiguration interface. + * + * @return A boolean value indicating whether Mockito should enable the class cache. It always returns false. + */ + @Override + public boolean enableClassCache() { + return false; + } +} \ No newline at end of file diff --git a/transaction-importer/transaction-importer-csv/src/test/resources/configuration/valid-config.json b/transaction-importer/transaction-importer-csv/src/test/resources/configuration/valid-config.json new file mode 100644 index 00000000..f9ade91f --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/test/resources/configuration/valid-config.json @@ -0,0 +1,22 @@ +{ + "has-headers": true, + "apply-rules": true, + "generate-accounts": false, + "date-format": "yyyyMMdd", + "delimiter": ",", + "column-roles": [ + "transaction-date", + "opposing-name", + "account-iban", + "opposing-iban", + "_ignore", + "custom-indicator", + "amount", + "_ignore", + "description" + ], + "custom-indicator": { + "deposit": "Bij", + "credit": "Af" + } +} diff --git a/transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-deposit.csv b/transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-deposit.csv new file mode 100644 index 00000000..1caf3629 --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-deposit.csv @@ -0,0 +1,2 @@ +Datum,Naam/Omschrijving,Rekening,Tegenrekening,Code,Af/Bij,Bedrag,Valuta,Mutatiesoort +20160531,Janssen PA,NL20INGB0001234567,NL69INGB0123454789,GT,Bij,"14,19",Internetbankieren,Naam: P. Post Omschrijving: Factuur 123 IBAN: NL69INGB0123456789 Kenmerk: 190451787399 \ No newline at end of file diff --git a/transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-withdrawal.csv b/transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-withdrawal.csv new file mode 100644 index 00000000..14a80150 --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/test/resources/csv-files/single-withdrawal.csv @@ -0,0 +1,2 @@ +Datum,Naam/Omschrijving,Rekening,Tegenrekening,Code,Af/Bij,Bedrag +20160531,MW GA Pieterse,NL20INGB0001234567,NL71INGB0009876543,GT,Af,"283,90",Internetbankieren,Naam: Mw G A Pieterse Omschrijving: inzake bestelling IBAN: NL71INGB0009876543 \ No newline at end of file diff --git a/transaction-importer/transaction-importer-csv/src/test/resources/logback.xml b/transaction-importer/transaction-importer-csv/src/test/resources/logback.xml new file mode 100644 index 00000000..f7605426 --- /dev/null +++ b/transaction-importer/transaction-importer-csv/src/test/resources/logback.xml @@ -0,0 +1,16 @@ + + + + false + + + %cyan(%d{HH:mm:ss.SSS}) %magenta[%-36X{correlationId}] %highlight(%-5level) %gray([%thread]) %magenta(%logger{36}) - %msg%n + + + + + + + +