From 6e97da7d20e3d6a5446c4dd180978bb75a855478 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 6 May 2024 22:10:44 +0200 Subject: [PATCH 01/26] core: added name to data source expression and value --- .../kleis/lcaac/core/lang/evaluator/ToValue.kt | 1 + .../lang/expression/DataSourceExpression.kt | 1 + .../lcaac/core/lang/value/DataSourceValue.kt | 1 + .../core/datasource/CsvSourceOperationsTest.kt | 4 ++++ .../reducer/DataExpressionReducerTest.kt | 17 +++++++++++++---- .../reducer/LcaExpressionReducerTest.kt | 1 + .../core/lang/evaluator/step/ReduceTest.kt | 1 + 7 files changed, 22 insertions(+), 4 deletions(-) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/ToValue.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/ToValue.kt index d137119a..a8fd5b6c 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/ToValue.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/ToValue.kt @@ -156,6 +156,7 @@ class ToValue( fun DataSourceExpression.toValue(): DataSourceValue { return when(this) { is EDataSource -> DataSourceValue( + this.name, this.location, this.schema.mapValues { it.value.toValue() }, this.filter.mapValues { it.value.toValue() }, diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/expression/DataSourceExpression.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/expression/DataSourceExpression.kt index fb9205ef..f701c53e 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/expression/DataSourceExpression.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/expression/DataSourceExpression.kt @@ -3,6 +3,7 @@ package ch.kleis.lcaac.core.lang.expression sealed interface DataSourceExpression data class EDataSource ( + val name: String, val location: String, val schema: Map>, val filter: Map> = emptyMap(), diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataSourceValue.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataSourceValue.kt index 15ad746f..fa8f25e6 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataSourceValue.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataSourceValue.kt @@ -1,6 +1,7 @@ package ch.kleis.lcaac.core.lang.value data class DataSourceValue( + val name: String, val location: String, val schema: Map>, val filter: Map> = emptyMap(), diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperationsTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperationsTest.kt index 63de15c5..41bd1527 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperationsTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperationsTest.kt @@ -43,6 +43,7 @@ class CsvSourceOperationsTest { mockFileLoader(), ) val source = DataSourceValue( + name = "source", location = "source.csv", schema = mapOf( "geo" to StringValue("FR"), @@ -87,6 +88,7 @@ class CsvSourceOperationsTest { mockFileLoader(), ) val source = DataSourceValue( + name = "source", location = "source.csv", schema = mapOf( "geo" to StringValue("FR"), @@ -122,6 +124,7 @@ class CsvSourceOperationsTest { mockFileLoader(), ) val source = DataSourceValue( + name = "source", location = "source.csv", schema = mapOf( "geo" to StringValue("FR"), @@ -154,6 +157,7 @@ class CsvSourceOperationsTest { mockFileLoader(), ) val source = DataSourceValue( + name = "source", location = "source.csv", schema = mapOf( "geo" to StringValue("FR"), diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducerTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducerTest.kt index 37c35166..45db298f 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducerTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducerTest.kt @@ -34,6 +34,7 @@ class DataExpressionReducerTest { fun reduceDataSource_withRefInSchema() { // given val dataSource = EDataSource( + name = "source", location = "source.csv", schema = mapOf( "col" to EQuantityScale(BasicNumber(2.0), EDataRef("foo")), @@ -51,6 +52,7 @@ class DataExpressionReducerTest { // then val expected = EDataSource( + name = "source", location = "source.csv", schema = mapOf( "col" to QuantityFixture.twoKilograms, @@ -63,6 +65,7 @@ class DataExpressionReducerTest { fun reduceDataSource_filterWithRef() { // given val dataSource = EDataSource( + name = "source", location = "source.csv", schema = mapOf( "id" to EStringLiteral("foo"), @@ -89,6 +92,7 @@ class DataExpressionReducerTest { fun reduceDataSource_dataSourceRef() { // given val dataSource = EDataSource( + name = "source", location = "source.csv", schema = mapOf( "volume" to QuantityFixture.oneLitre, @@ -110,6 +114,7 @@ class DataExpressionReducerTest { fun reduceDataSource_filter() { // given val dataSource = EDataSource( + name = "source", location = "source.csv", schema = mapOf( "label" to EStringLiteral("value"), @@ -137,7 +142,7 @@ class DataExpressionReducerTest { @Test fun reduceDataSource_filter_accumulation() { // given - val dataSource = EDataSource(location = "source.csv", schema = mapOf( + val dataSource = EDataSource(name = "source", location = "source.csv", schema = mapOf( "label" to EStringLiteral("value"), "geo" to EStringLiteral("FR"), "volume" to QuantityFixture.oneLitre, @@ -161,7 +166,7 @@ class DataExpressionReducerTest { @Test fun reduceDataSource_filter_whenInvalid() { // given - val dataSource = EDataSource(location = "source.csv", schema = mapOf( + val dataSource = EDataSource(name = "source", location = "source.csv", schema = mapOf( "volume" to QuantityFixture.oneLitre, "mass" to QuantityFixture.oneKilogram, )) @@ -183,6 +188,7 @@ class DataExpressionReducerTest { // given val expression = ESumProduct(EDataSourceRef("source"), listOf("volume", "mass")) val dataSource = EDataSource( + name = "source", location = "source.csv", schema = mapOf( "volume" to QuantityFixture.oneLitre, @@ -220,6 +226,7 @@ class DataExpressionReducerTest { // given val expression = ESumProduct(EDataSourceRef("foo"), listOf("mass")) val dataSource = EDataSource( + name = "source", location = "source.csv", schema = mapOf( "mass" to QuantityFixture.oneKilogram, @@ -250,6 +257,7 @@ class DataExpressionReducerTest { @Test fun reduce_whenFirstRecordOfDataSourceRef() { val dataSource = EDataSource( + name = "source", location = "source.csv", schema = mapOf( "mass" to QuantityFixture.oneKilogram @@ -283,7 +291,7 @@ class DataExpressionReducerTest { @Test fun reduce_whenDefaultRecordOfDataSourceRef() { // given - val dataSource = EDataSource(location = "source.csv", schema = mapOf( + val dataSource = EDataSource(name = "source", location = "source.csv", schema = mapOf( "x" to EDataRef("a"), )) val record = EDefaultRecordOf(EDataSourceRef("source")) @@ -311,7 +319,7 @@ class DataExpressionReducerTest { @Test fun reduce_whenDefaultRecordOfDataSourceRef_invalidRef() { // given - val dataSource = EDataSource(location = "source.csv", schema = mapOf( + val dataSource = EDataSource(name="source", location = "source.csv", schema = mapOf( "x" to EDataRef("a"), )) val record = EDefaultRecordOf(EDataSourceRef("foo")) @@ -972,6 +980,7 @@ class DataExpressionReducerTest { fun reduce_whenUnitOf_withDataSourceExpression() { // given val dataSource = EDataSource( + name = "source", location = "source.csv", schema = mapOf( "n_items" to EQuantityScale(BasicNumber(0.0), UnitFixture.unit), diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducerTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducerTest.kt index 82045c48..f41ac420 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducerTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducerTest.kt @@ -16,6 +16,7 @@ class LcaExpressionReducerTest { private val ops = BasicOperations private val sourceOps = mockk>() private val emptyDataSource = EDataSource( + name = "source", "location.csv", emptyMap(), emptyMap() diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/step/ReduceTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/step/ReduceTest.kt index 1108a57a..111ff84b 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/step/ReduceTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/step/ReduceTest.kt @@ -38,6 +38,7 @@ class ReduceTest { ) ) val dataSource = EDataSource( + name = "source", location = "foo.csv", schema = mapOf( "mass" to QuantityFixture.oneLitre From 4f8fbf0254843347dda99d85a72e0e9ac5f9b9b6 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 6 May 2024 22:12:47 +0200 Subject: [PATCH 02/26] grammar: parse datasource name --- grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt | 5 +++-- .../src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt | 2 +- grammar/src/test/kotlin/ch/kleis/lcaac/grammar/LoaderTest.kt | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt b/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt index c8c9c1a2..3f236411 100644 --- a/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt +++ b/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt @@ -374,8 +374,9 @@ class CoreMapper( key to value } return EDataSource( - location, - schema, + name = name, + location = location, + schema = schema, ) } } diff --git a/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt b/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt index 95d44810..8caad2a3 100644 --- a/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt +++ b/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt @@ -207,7 +207,7 @@ class CoreMapperTest { val actual = mapper.dataSourceDefinition(ctx) // then - val expected = EDataSource(location = "file.csv", schema = mapOf( + val expected = EDataSource(name = "source", location = "file.csv", schema = mapOf( "mass" to EQuantityScale(BasicNumber(1.0), EDataRef("kg")), "geo" to EStringLiteral("FR"), )) diff --git a/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/LoaderTest.kt b/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/LoaderTest.kt index 398395e9..6e299ff0 100644 --- a/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/LoaderTest.kt +++ b/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/LoaderTest.kt @@ -75,6 +75,7 @@ class LoaderTest { // then val expected = EDataSource( + name = "source", location = "file.csv", schema = mapOf( "mass" to EQuantityScale(BasicNumber(1.0), EDataRef("kg")), @@ -105,6 +106,7 @@ class LoaderTest { // then val expected = EDataSource( + name = "source", location = "file.csv", schema = mapOf( "mass" to EQuantityScale(BasicNumber(1.0), EDataRef("kg")), From 7623258944023b56127c2c98e1c67a37860e717a Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Tue, 7 May 2024 16:40:45 +0200 Subject: [PATCH 03/26] core: datasources: simplify classes --- .../core/datasource/CsvSourceOperations.kt | 99 ++++++++++++++----- .../core/datasource/DataSourceDescription.kt | 8 -- .../datasource/DataSourceOperationsBase.kt | 65 ------------ 3 files changed, 74 insertions(+), 98 deletions(-) delete mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceDescription.kt delete mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceOperationsBase.kt diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperations.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperations.kt index 2a4bc086..2256eefc 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperations.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperations.kt @@ -1,13 +1,16 @@ package ch.kleis.lcaac.core.datasource import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException -import ch.kleis.lcaac.core.lang.expression.DataExpression -import ch.kleis.lcaac.core.lang.expression.EQuantityScale -import ch.kleis.lcaac.core.lang.expression.ERecord -import ch.kleis.lcaac.core.lang.expression.EStringLiteral +import ch.kleis.lcaac.core.lang.evaluator.ToValue +import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer +import ch.kleis.lcaac.core.lang.expression.* +import ch.kleis.lcaac.core.lang.register.DataSourceRegister +import ch.kleis.lcaac.core.lang.value.DataSourceValue +import ch.kleis.lcaac.core.lang.value.DataValue import ch.kleis.lcaac.core.lang.value.QuantityValue import ch.kleis.lcaac.core.lang.value.StringValue import ch.kleis.lcaac.core.math.QuantityOperations +import ch.kleis.lcaac.core.prelude.Prelude import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVParser import java.io.File @@ -22,30 +25,76 @@ class CsvSourceOperations( val location = Paths.get(path.absolutePath, it) location.toFile().inputStream() } -) : DataSourceOperationsBase(ops, { description -> - val inputStream = fileLoader(description.location) - val parser = CSVParser(inputStream.reader(), format) - val header = parser.headerMap - val schema = description.schema - parser.iterator().asSequence() - .map { csvRecord -> - val entries = header - .filter { entry -> schema.containsKey(entry.key) } - .mapValues { entry -> - val columnDefaultValue = schema[entry.key]!! - val position = entry.value - val element = csvRecord[position] - when (columnDefaultValue) { - is QuantityValue -> parseQuantityWithDefaultUnit(ops, element, columnDefaultValue.unit.toEUnitLiteral()) - is StringValue -> EStringLiteral(element) - else -> throw IllegalStateException( - "invalid schema: column '${entry.key}' has an invalid default value" - ) +) : DataSourceOperations { + private fun load(location: String, schema: Map>): Sequence> { + val inputStream = fileLoader(location) + val parser = CSVParser(inputStream.reader(), format) + val header = parser.headerMap + return parser.iterator().asSequence() + .map { csvRecord -> + val entries = header + .filter { entry -> schema.containsKey(entry.key) } + .mapValues { entry -> + val columnDefaultValue = schema[entry.key]!! + val position = entry.value + val element = csvRecord[position] + when (columnDefaultValue) { + is QuantityValue -> parseQuantityWithDefaultUnit(ops, element, columnDefaultValue.unit.toEUnitLiteral()) + is StringValue -> EStringLiteral(element) + else -> throw IllegalStateException( + "invalid schema: column '${entry.key}' has an invalid default value" + ) + } } + ERecord(entries) + } + } + + override fun readAll(source: DataSourceValue): Sequence> { + val records = load(source.location, source.schema) + val filter = source.filter + return records + .filter { record -> + filter.entries.all { + val expected = it.value + if (expected is StringValue) { + val actual = record.entries[it.key] + ?.let { with(ToValue(ops)) { it.toValue() } } + ?: throw IllegalStateException( + "${source.location}: invalid schema: unknown column '${it.key}'" + ) + actual == expected + } else throw EvaluatorException("invalid matching condition $it") } - ERecord(entries) + } + } + + override fun sumProduct(source: DataSourceValue, columns: List): DataExpression { + val reducer = DataExpressionReducer( + dataRegister = Prelude.units(), + dataSourceRegister = DataSourceRegister.empty(), + ops = ops, + sourceOps = this, + ) + return readAll(source).map { record -> + columns.map { column -> + record.entries[column] + ?: throw IllegalStateException( + "${source.location}: invalid schema: unknown column '$column'" + ) + }.reduce { acc, expression -> + reducer.reduce(EQuantityMul(acc, expression)) + } + }.reduce { acc, expression -> + reducer.reduce(EQuantityAdd(acc, expression)) } -}) + } + + override fun getFirst(source: DataSourceValue): ERecord { + return readAll(source).firstOrNull() + ?: throw EvaluatorException("no record found in '${source.location}' matching ${source.filter}") + } +} private val format = CSVFormat.DEFAULT.builder() .setHeader() diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceDescription.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceDescription.kt deleted file mode 100644 index 9186ad45..00000000 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceDescription.kt +++ /dev/null @@ -1,8 +0,0 @@ -package ch.kleis.lcaac.core.datasource - -import ch.kleis.lcaac.core.lang.value.DataValue - -data class DataSourceDescription( - val location: String, - val schema: Map>, -) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceOperationsBase.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceOperationsBase.kt deleted file mode 100644 index 0c217106..00000000 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceOperationsBase.kt +++ /dev/null @@ -1,65 +0,0 @@ -package ch.kleis.lcaac.core.datasource - -import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException -import ch.kleis.lcaac.core.lang.evaluator.ToValue -import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer -import ch.kleis.lcaac.core.lang.expression.DataExpression -import ch.kleis.lcaac.core.lang.expression.EQuantityAdd -import ch.kleis.lcaac.core.lang.expression.EQuantityMul -import ch.kleis.lcaac.core.lang.expression.ERecord -import ch.kleis.lcaac.core.lang.register.DataSourceRegister -import ch.kleis.lcaac.core.lang.value.DataSourceValue -import ch.kleis.lcaac.core.lang.value.StringValue -import ch.kleis.lcaac.core.math.QuantityOperations -import ch.kleis.lcaac.core.prelude.Prelude - -open class DataSourceOperationsBase( - private val ops: QuantityOperations, - private val load: (DataSourceDescription) -> Sequence>, -) : DataSourceOperations { - override fun readAll(source: DataSourceValue): Sequence> { - val description = DataSourceDescription(source.location, source.schema) - val records = load(description) - val filter = source.filter - return records - .filter { record -> - filter.entries.all { - val expected = it.value - if (expected is StringValue) { - val actual = record.entries[it.key] - ?.let { with(ToValue(ops)) { it.toValue() } } - ?: throw IllegalStateException( - "${source.location}: invalid schema: unknown column '${it.key}'" - ) - actual == expected - } else throw EvaluatorException("invalid matching condition $it") - } - } - } - - override fun sumProduct(source: DataSourceValue, columns: List): DataExpression { - val reducer = DataExpressionReducer( - dataRegister = Prelude.units(), - dataSourceRegister = DataSourceRegister.empty(), - ops = ops, - sourceOps = this, - ) - return readAll(source).map { record -> - columns.map { column -> - record.entries[column] - ?: throw IllegalStateException( - "${source.location}: invalid schema: unknown column '$column'" - ) - }.reduce { acc, expression -> - reducer.reduce(EQuantityMul(acc, expression)) - } - }.reduce { acc, expression -> - reducer.reduce(EQuantityAdd(acc, expression)) - } - } - - override fun getFirst(source: DataSourceValue): ERecord { - return readAll(source).firstOrNull() - ?: throw EvaluatorException("no record found in '${source.location}' matching ${source.filter}") - } -} From 6cb7df48476666b26ec3072d57f67c82a933d4d9 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Tue, 7 May 2024 23:44:38 +0200 Subject: [PATCH 04/26] core: refactor: connector + manager --- .../ch/kleis/lcaac/core/config/LcaacConfig.kt | 9 + .../lcaac/core/config/LcaacConnectorConfig.kt | 6 + .../core/config/LcaacDataSourceConfig.kt | 9 + .../core/datasource/DataSourceConnector.kt | 11 + .../core/datasource/DataSourceManager.kt | 67 ++++++ .../core/datasource/DataSourceOperations.kt | 5 +- .../core/datasource/DummySourceOperations.kt | 2 +- .../CsvConnector.kt} | 57 ++---- .../core/datasource/csv/CsvConnectorConfig.kt | 28 +++ .../evaluator/reducer/LcaExpressionReducer.kt | 6 +- .../reducer/TemplateExpressionReducer.kt | 2 +- .../lang/expression/DataSourceExpression.kt | 2 +- .../lcaac/core/lang/value/DataSourceValue.kt | 2 +- ...eOperationsTest.kt => CsvConnectorTest.kt} | 64 ++---- .../core/datasource/DataSourceManagerTest.kt | 190 ++++++++++++++++++ .../reducer/LcaExpressionReducerTest.kt | 12 +- 16 files changed, 376 insertions(+), 96 deletions(-) create mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt create mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConnectorConfig.kt create mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacDataSourceConfig.kt create mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceConnector.kt create mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt rename core/src/main/kotlin/ch/kleis/lcaac/core/datasource/{CsvSourceOperations.kt => csv/CsvConnector.kt} (63%) create mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt rename core/src/test/kotlin/ch/kleis/lcaac/core/datasource/{CsvSourceOperationsTest.kt => CsvConnectorTest.kt} (75%) create mode 100644 core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManagerTest.kt diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt new file mode 100644 index 00000000..d2006489 --- /dev/null +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt @@ -0,0 +1,9 @@ +package ch.kleis.lcaac.core.config + +data class LcaacConfig( + val name: String, + val description: String, + val datasources: Map, + val connectors: Map, +) + diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConnectorConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConnectorConfig.kt new file mode 100644 index 00000000..33304ff9 --- /dev/null +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConnectorConfig.kt @@ -0,0 +1,6 @@ +package ch.kleis.lcaac.core.config + +data class LcaacConnectorConfig( + val name: String, + val options: Map, +) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacDataSourceConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacDataSourceConfig.kt new file mode 100644 index 00000000..80f5ea6d --- /dev/null +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacDataSourceConfig.kt @@ -0,0 +1,9 @@ +package ch.kleis.lcaac.core.config + +data class LcaacDataSourceConfig( + val name: String, + val connector: String = "csv", + val location: String = "$name.csv", + val primaryKey: String = "id", + val options: Map = emptyMap(), +) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceConnector.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceConnector.kt new file mode 100644 index 00000000..c12cb725 --- /dev/null +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceConnector.kt @@ -0,0 +1,11 @@ +package ch.kleis.lcaac.core.datasource + +import ch.kleis.lcaac.core.config.LcaacConnectorConfig +import ch.kleis.lcaac.core.config.LcaacDataSourceConfig +import ch.kleis.lcaac.core.lang.expression.ERecord +import ch.kleis.lcaac.core.lang.value.DataSourceValue + +interface DataSourceConnector { + fun getFirst(config: LcaacDataSourceConfig, source: DataSourceValue): ERecord + fun getAll(config: LcaacDataSourceConfig, source: DataSourceValue): Sequence> +} diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt new file mode 100644 index 00000000..63a84c88 --- /dev/null +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt @@ -0,0 +1,67 @@ +package ch.kleis.lcaac.core.datasource + +import ch.kleis.lcaac.core.config.LcaacConfig +import ch.kleis.lcaac.core.config.LcaacDataSourceConfig +import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer +import ch.kleis.lcaac.core.lang.expression.DataExpression +import ch.kleis.lcaac.core.lang.expression.EQuantityAdd +import ch.kleis.lcaac.core.lang.expression.EQuantityMul +import ch.kleis.lcaac.core.lang.expression.ERecord +import ch.kleis.lcaac.core.lang.register.DataSourceRegister +import ch.kleis.lcaac.core.lang.value.DataSourceValue +import ch.kleis.lcaac.core.math.QuantityOperations +import ch.kleis.lcaac.core.prelude.Prelude + +class DataSourceManager( + private val config: LcaacConfig, + private val ops: QuantityOperations, +) : DataSourceOperations { + private val connectors = HashMap>() + + fun registerConnector(connectorName: String, connector: DataSourceConnector) { + connectors[connectorName] = connector + } + + private fun configOf(name: String): LcaacDataSourceConfig { + return config.datasources[name] + ?: throw IllegalArgumentException("No configuration found for datasource '$name'") + } + + private fun connectorOf(config: LcaacDataSourceConfig): DataSourceConnector { + return connectors[config.connector] + ?: throw IllegalArgumentException("Unknown connect '${config.connector}'") + } + + override fun getFirst(source: DataSourceValue): ERecord { + val sourceConfig = configOf(source.name) + val connector = connectorOf(sourceConfig) + return connector.getFirst(sourceConfig, source) + } + + override fun getAll(source: DataSourceValue): Sequence> { + val sourceConfig = configOf(source.name) + val connector = connectorOf(sourceConfig) + return connector.getAll(sourceConfig, source) + } + + override fun sumProduct(source: DataSourceValue, columns: List): DataExpression { + val reducer = DataExpressionReducer( + dataRegister = Prelude.units(), + dataSourceRegister = DataSourceRegister.empty(), + ops = ops, + sourceOps = this, + ) + return getAll(source).map { record -> + columns.map { column -> + record.entries[column] + ?: throw IllegalStateException( + "${source.location}: invalid schema: unknown column '$column'" + ) + }.reduce { acc, expression -> + reducer.reduce(EQuantityMul(acc, expression)) + } + }.reduce { acc, expression -> + reducer.reduce(EQuantityAdd(acc, expression)) + } + } +} diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceOperations.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceOperations.kt index f70bdd8c..6437166e 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceOperations.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceOperations.kt @@ -5,7 +5,8 @@ import ch.kleis.lcaac.core.lang.expression.ERecord import ch.kleis.lcaac.core.lang.value.DataSourceValue interface DataSourceOperations { - fun readAll(source: DataSourceValue): Sequence> - fun sumProduct(source: DataSourceValue, columns: List): DataExpression fun getFirst(source: DataSourceValue): ERecord + fun getAll(source: DataSourceValue): Sequence> + fun sumProduct(source: DataSourceValue, columns: List): DataExpression } + diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DummySourceOperations.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DummySourceOperations.kt index 32f99c86..06c3cc0e 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DummySourceOperations.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DummySourceOperations.kt @@ -14,7 +14,7 @@ import ch.kleis.lcaac.core.lang.value.* */ class DummySourceOperations : DataSourceOperations { - override fun readAll(source: DataSourceValue): Sequence> { + override fun getAll(source: DataSourceValue): Sequence> { return sequenceOf(ERecord(source.schema.mapValues { it.value.toDataExpression() })) } diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperations.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt similarity index 63% rename from core/src/main/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperations.kt rename to core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt index 2256eefc..9fdbf227 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperations.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt @@ -1,31 +1,33 @@ -package ch.kleis.lcaac.core.datasource +package ch.kleis.lcaac.core.datasource.csv +import ch.kleis.lcaac.core.config.LcaacConnectorConfig +import ch.kleis.lcaac.core.config.LcaacDataSourceConfig +import ch.kleis.lcaac.core.datasource.DataSourceConnector import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException import ch.kleis.lcaac.core.lang.evaluator.ToValue -import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer -import ch.kleis.lcaac.core.lang.expression.* -import ch.kleis.lcaac.core.lang.register.DataSourceRegister +import ch.kleis.lcaac.core.lang.expression.DataExpression +import ch.kleis.lcaac.core.lang.expression.EQuantityScale +import ch.kleis.lcaac.core.lang.expression.ERecord +import ch.kleis.lcaac.core.lang.expression.EStringLiteral import ch.kleis.lcaac.core.lang.value.DataSourceValue import ch.kleis.lcaac.core.lang.value.DataValue import ch.kleis.lcaac.core.lang.value.QuantityValue import ch.kleis.lcaac.core.lang.value.StringValue import ch.kleis.lcaac.core.math.QuantityOperations -import ch.kleis.lcaac.core.prelude.Prelude import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVParser -import java.io.File import java.io.InputStream import java.lang.Double.parseDouble import java.nio.file.Paths -class CsvSourceOperations( - private val path: File, +class CsvConnector( + private val connectorConfig: CsvConnectorConfig, private val ops: QuantityOperations, - private val fileLoader: (String) -> InputStream = { - val location = Paths.get(path.absolutePath, it) - location.toFile().inputStream() + private val fileLoader: (String) -> InputStream = { location -> + val csvFile = Paths.get(connectorConfig.directory.absolutePath, location) + csvFile.toFile().inputStream() } -) : DataSourceOperations { +) : DataSourceConnector { private fun load(location: String, schema: Map>): Sequence> { val inputStream = fileLoader(location) val parser = CSVParser(inputStream.reader(), format) @@ -50,8 +52,8 @@ class CsvSourceOperations( } } - override fun readAll(source: DataSourceValue): Sequence> { - val records = load(source.location, source.schema) + override fun getAll(config: LcaacDataSourceConfig, source: DataSourceValue): Sequence> { + val records = load(config.location, source.schema) val filter = source.filter return records .filter { record -> @@ -69,30 +71,9 @@ class CsvSourceOperations( } } - override fun sumProduct(source: DataSourceValue, columns: List): DataExpression { - val reducer = DataExpressionReducer( - dataRegister = Prelude.units(), - dataSourceRegister = DataSourceRegister.empty(), - ops = ops, - sourceOps = this, - ) - return readAll(source).map { record -> - columns.map { column -> - record.entries[column] - ?: throw IllegalStateException( - "${source.location}: invalid schema: unknown column '$column'" - ) - }.reduce { acc, expression -> - reducer.reduce(EQuantityMul(acc, expression)) - } - }.reduce { acc, expression -> - reducer.reduce(EQuantityAdd(acc, expression)) - } - } - - override fun getFirst(source: DataSourceValue): ERecord { - return readAll(source).firstOrNull() - ?: throw EvaluatorException("no record found in '${source.location}' matching ${source.filter}") + override fun getFirst(config: LcaacDataSourceConfig, source: DataSourceValue): ERecord { + return getAll(config, source).firstOrNull() + ?: throw EvaluatorException("no record found in '${config.location}' matching ${source.filter}") } } diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt new file mode 100644 index 00000000..6174573c --- /dev/null +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt @@ -0,0 +1,28 @@ +package ch.kleis.lcaac.core.datasource.csv + +import ch.kleis.lcaac.core.config.LcaacConnectorConfig +import ch.kleis.lcaac.core.datasource.csv.CsvConnectorConfig.Companion.CSV_CONNECTOR_KEY_DIRECTORY +import ch.kleis.lcaac.core.datasource.csv.CsvConnectorConfig.Companion.CSV_CONNECTOR_NAME +import java.io.File +import java.nio.file.Path + +data class CsvConnectorConfig( + val directory: File, +) { + companion object { + const val CSV_CONNECTOR_NAME = "csv" + const val CSV_CONNECTOR_KEY_DIRECTORY = "directory" + } +} + +fun LcaacConnectorConfig.csv(): CsvConnectorConfig? { + if (this.name != CSV_CONNECTOR_NAME) { + return null + } + val directory = this.options[CSV_CONNECTOR_KEY_DIRECTORY] + ?.let { Path.of(it).toFile() } + ?: return null + return CsvConnectorConfig( + directory, + ) +} diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducer.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducer.kt index 4eb5a067..9ec4dd79 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducer.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducer.kt @@ -84,7 +84,7 @@ class LcaExpressionReducer( is EBlockForEach -> { val ds = dataExpressionReducer.evalDataSource(expression.dataSource) - sourceOps.readAll(ds) + sourceOps.getAll(ds) .map { dataExpressionReducer.reduce(it) } .flatMap { record -> val reducer = push(mapOf( @@ -110,7 +110,7 @@ class LcaExpressionReducer( is EBlockForEach -> { val ds = dataExpressionReducer.evalDataSource(expression.dataSource) - sourceOps.readAll(ds) + sourceOps.getAll(ds) .map { dataExpressionReducer.reduce(it) } .flatMap { record -> val reducer = push(mapOf( @@ -136,7 +136,7 @@ class LcaExpressionReducer( is EBlockForEach -> { val ds = dataExpressionReducer.evalDataSource(expression.dataSource) - sourceOps.readAll(ds) + sourceOps.getAll(ds) .map { dataExpressionReducer.reduce(it) } .flatMap { record -> val reducer = push(mapOf( diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/TemplateExpressionReducer.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/TemplateExpressionReducer.kt index 5613ac94..58a5a475 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/TemplateExpressionReducer.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/TemplateExpressionReducer.kt @@ -15,7 +15,7 @@ class TemplateExpressionReducer( private val ops: QuantityOperations, private val sourceOps: DataSourceOperations, dataRegister: DataRegister = DataRegister.empty(), - dataSourceRegister: DataSourceRegister = DataSourceRegister.empty(), + dataSourceRegister: DataSourceRegister = DataSourceRegister.empty(), ) { private val dataRegister = Register(dataRegister) private val dataSourceRegister = Register(dataSourceRegister) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/expression/DataSourceExpression.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/expression/DataSourceExpression.kt index f701c53e..e96d92f5 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/expression/DataSourceExpression.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/expression/DataSourceExpression.kt @@ -4,7 +4,7 @@ sealed interface DataSourceExpression data class EDataSource ( val name: String, - val location: String, + @Deprecated("will be removed") val location: String, val schema: Map>, val filter: Map> = emptyMap(), ) : DataSourceExpression diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataSourceValue.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataSourceValue.kt index fa8f25e6..ac3fa335 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataSourceValue.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataSourceValue.kt @@ -2,7 +2,7 @@ package ch.kleis.lcaac.core.lang.value data class DataSourceValue( val name: String, - val location: String, + @Deprecated("will be removed") val location: String, val schema: Map>, val filter: Map> = emptyMap(), ) diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperationsTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvConnectorTest.kt similarity index 75% rename from core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperationsTest.kt rename to core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvConnectorTest.kt index 41bd1527..622303b3 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperationsTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvConnectorTest.kt @@ -1,5 +1,7 @@ package ch.kleis.lcaac.core.datasource +import ch.kleis.lcaac.core.config.LcaacDataSourceConfig +import ch.kleis.lcaac.core.datasource.csv.CsvConnector import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException import ch.kleis.lcaac.core.lang.expression.EQuantityScale import ch.kleis.lcaac.core.lang.expression.ERecord @@ -18,7 +20,7 @@ import org.junit.jupiter.api.assertThrows import java.io.InputStream import kotlin.test.assertEquals -class CsvSourceOperationsTest { +class CsvConnectorTest { private fun mockFileLoader(): (String) -> InputStream { val content = """ geo,n_items,mass @@ -35,9 +37,9 @@ class CsvSourceOperationsTest { private val ops = BasicOperations @Test - fun readAll() { + fun getAll() { // given - val sourceOps = CsvSourceOperations( + val connector = CsvConnector( mockk(), ops, mockFileLoader(), @@ -54,9 +56,13 @@ class CsvSourceOperationsTest { "geo" to StringValue("FR") ) ) + val config = LcaacDataSourceConfig( + name = "source", + location = "source.csv", + ) // when - val actual = sourceOps.readAll(source).toList() + val actual = connector.getAll(config, source).toList() // then val expected = listOf( @@ -80,9 +86,9 @@ class CsvSourceOperationsTest { } @Test - fun sumProduct() { + fun getFirst() { // given - val sourceOps = CsvSourceOperations( + val connector = CsvConnector( mockk(), ops, mockFileLoader(), @@ -96,48 +102,16 @@ class CsvSourceOperationsTest { "mass" to QuantityValueFixture.oneKilogram, ), filter = mapOf( - "geo" to StringValue("FR") + "geo" to StringValue("UK") ) ) - - // when - val actual = sourceOps.sumProduct(source, listOf("n_items", "mass")) - - // then - val expected = EQuantityScale( - BasicNumber(7.0), - EUnitLiteral( - UnitFixture.kg.symbol.multiply(UnitFixture.unit.symbol), - 1.0, - UnitFixture.kg.dimension.multiply(UnitFixture.unit.dimension), - ), - ) - assertEquals(expected, actual) - } - - @Test - fun getFirst() { - // given - val sourceOps = CsvSourceOperations( - mockk(), - ops, - mockFileLoader(), - ) - val source = DataSourceValue( + val config = LcaacDataSourceConfig( name = "source", location = "source.csv", - schema = mapOf( - "geo" to StringValue("FR"), - "n_items" to QuantityValueFixture.oneUnit, - "mass" to QuantityValueFixture.oneKilogram, - ), - filter = mapOf( - "geo" to StringValue("UK") - ) ) // when - val actual = sourceOps.getFirst(source) + val actual = connector.getFirst(config, source) // then val expected = ERecord(mapOf( @@ -151,7 +125,7 @@ class CsvSourceOperationsTest { @Test fun getFirst_whenEmpty_throwEvaluatorException() { // given - val sourceOps = CsvSourceOperations( + val connector = CsvConnector( mockk(), ops, mockFileLoader(), @@ -168,9 +142,13 @@ class CsvSourceOperationsTest { "geo" to StringValue("A_NON_EXISTING_COUNTRY") ) ) + val config = LcaacDataSourceConfig( + name = "source", + location = "source.csv", + ) // when/then - val e = assertThrows { sourceOps.getFirst(source) } + val e = assertThrows { connector.getFirst(config, source) } assertEquals("no record found in 'source.csv' matching {geo=A_NON_EXISTING_COUNTRY}", e.message) } } diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManagerTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManagerTest.kt new file mode 100644 index 00000000..4a371364 --- /dev/null +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManagerTest.kt @@ -0,0 +1,190 @@ +package ch.kleis.lcaac.core.datasource + +import ch.kleis.lcaac.core.config.LcaacConfig +import ch.kleis.lcaac.core.config.LcaacConnectorConfig +import ch.kleis.lcaac.core.config.LcaacDataSourceConfig +import ch.kleis.lcaac.core.lang.expression.EQuantityScale +import ch.kleis.lcaac.core.lang.expression.ERecord +import ch.kleis.lcaac.core.lang.expression.EStringLiteral +import ch.kleis.lcaac.core.lang.expression.EUnitLiteral +import ch.kleis.lcaac.core.lang.fixture.QuantityFixture +import ch.kleis.lcaac.core.lang.fixture.QuantityValueFixture +import ch.kleis.lcaac.core.lang.fixture.UnitFixture +import ch.kleis.lcaac.core.lang.value.DataSourceValue +import ch.kleis.lcaac.core.lang.value.StringValue +import ch.kleis.lcaac.core.math.basic.BasicNumber +import ch.kleis.lcaac.core.math.basic.BasicOperations +import io.mockk.every +import io.mockk.mockk +import kotlin.test.Test +import kotlin.test.assertEquals + +class DataSourceManagerTest { + private val connectorName = "connector" + private val connectorConfig = LcaacConnectorConfig(name = connectorName, options = emptyMap()) + + private val sourceName = "source" + private val sourceConfig = LcaacDataSourceConfig(name = sourceName, connector = connectorName) + + private val config = LcaacConfig( + name = "project", + description = "description", + datasources = mapOf( + sourceName to sourceConfig + ), + connectors = mapOf( + connectorName to connectorConfig + ), + ) + private val ops = BasicOperations + + @Test + fun getAll() { + // given + val connector = mockk>() + every { connector.getAll(any(), any()) } returns sequenceOf( + ERecord(mapOf( + "geo" to EStringLiteral("FR"), + "n_items" to QuantityFixture.oneUnit, + "mass" to QuantityFixture.oneKilogram, + )), + ERecord(mapOf( + "geo" to EStringLiteral("FR"), + "n_items" to QuantityFixture.twoUnits, + "mass" to QuantityFixture.twoKilograms, + )), + ERecord(mapOf( + "geo" to EStringLiteral("FR"), + "n_items" to QuantityFixture.oneUnit, + "mass" to QuantityFixture.twoKilograms, + )), + ) + + val sourceOps = DataSourceManager(config, ops) + sourceOps.registerConnector(connectorName, connector) + val source = DataSourceValue( + name = sourceName, + location = "source.csv", + schema = mapOf( + "geo" to StringValue("FR"), + "n_items" to QuantityValueFixture.oneUnit, + "mass" to QuantityValueFixture.oneKilogram, + ), + filter = mapOf( + "geo" to StringValue("FR") + ) + ) + + // when + val actual = sourceOps.getAll(source).toList() + + // then + val expected = listOf( + ERecord(mapOf( + "geo" to EStringLiteral("FR"), + "n_items" to QuantityFixture.oneUnit, + "mass" to QuantityFixture.oneKilogram, + )), + ERecord(mapOf( + "geo" to EStringLiteral("FR"), + "n_items" to QuantityFixture.twoUnits, + "mass" to QuantityFixture.twoKilograms, + )), + ERecord(mapOf( + "geo" to EStringLiteral("FR"), + "n_items" to QuantityFixture.oneUnit, + "mass" to QuantityFixture.twoKilograms, + )), + ) + assertEquals(expected, actual) + } + + @Test + fun getFirst() { + // given + val connector = mockk>() + every { connector.getFirst(any(), any()) } returns ERecord(mapOf( + "geo" to EStringLiteral("FR"), + "n_items" to QuantityFixture.oneUnit, + "mass" to QuantityFixture.oneKilogram, + )) + + val sourceOps = DataSourceManager(config, ops) + sourceOps.registerConnector(connectorName, connector) + val source = DataSourceValue( + name = sourceName, + location = "source.csv", + schema = mapOf( + "geo" to StringValue("FR"), + "n_items" to QuantityValueFixture.oneUnit, + "mass" to QuantityValueFixture.oneKilogram, + ), + filter = mapOf( + "geo" to StringValue("FR") + ) + ) + + // when + val actual = sourceOps.getFirst(source) + + // then + val expected = ERecord(mapOf( + "geo" to EStringLiteral("FR"), + "n_items" to QuantityFixture.oneUnit, + "mass" to QuantityFixture.oneKilogram, + )) + assertEquals(expected, actual) + } + + @Test + fun sumProduct() { + // given + val connector = mockk>() + every { connector.getAll(any(), any()) } returns sequenceOf( + ERecord(mapOf( + "geo" to EStringLiteral("FR"), + "n_items" to QuantityFixture.oneUnit, + "mass" to QuantityFixture.oneKilogram, + )), + ERecord(mapOf( + "geo" to EStringLiteral("FR"), + "n_items" to QuantityFixture.twoUnits, + "mass" to QuantityFixture.twoKilograms, + )), + ERecord(mapOf( + "geo" to EStringLiteral("FR"), + "n_items" to QuantityFixture.oneUnit, + "mass" to QuantityFixture.twoKilograms, + )), + ) + + val sourceOps = DataSourceManager(config, ops) + sourceOps.registerConnector(connectorName, connector) + val source = DataSourceValue( + name = sourceName, + location = "source.csv", + schema = mapOf( + "geo" to StringValue("FR"), + "n_items" to QuantityValueFixture.oneUnit, + "mass" to QuantityValueFixture.oneKilogram, + ), + filter = mapOf( + "geo" to StringValue("FR") + ) + ) + + // when + val actual = sourceOps.sumProduct(source, listOf("n_items", "mass")) + + // then + val expected = EQuantityScale( + BasicNumber(7.0), + EUnitLiteral( + UnitFixture.kg.symbol.multiply(UnitFixture.unit.symbol), + 1.0, + UnitFixture.kg.dimension.multiply(UnitFixture.unit.dimension), + ), + ) + assertEquals(expected, actual) + } +} diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducerTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducerTest.kt index f41ac420..cee4495c 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducerTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducerTest.kt @@ -47,7 +47,7 @@ class LcaExpressionReducerTest { impacts = listOf(block), ) val sourceOps = mockk>() - every { sourceOps.readAll(any()) } returns sequenceOf( + every { sourceOps.getAll(any()) } returns sequenceOf( ERecord(mapOf( "mass" to QuantityFixture.oneKilogram, )), @@ -109,7 +109,7 @@ class LcaExpressionReducerTest { biosphere = listOf(block), ) val sourceOps = mockk>() - every { sourceOps.readAll(any()) } returns sequenceOf( + every { sourceOps.getAll(any()) } returns sequenceOf( ERecord(mapOf( "mass" to QuantityFixture.oneKilogram, )), @@ -171,7 +171,7 @@ class LcaExpressionReducerTest { inputs = listOf(block), ) val sourceOps = mockk>() - every { sourceOps.readAll(any()) } returns sequenceOf( + every { sourceOps.getAll(any()) } returns sequenceOf( ERecord(mapOf( "mass" to QuantityFixture.oneKilogram, )), @@ -233,7 +233,7 @@ class LcaExpressionReducerTest { inputs = listOf(block), ) val sourceOps = mockk>() - every { sourceOps.readAll(any()) } returns sequenceOf( + every { sourceOps.getAll(any()) } returns sequenceOf( ERecord(mapOf( "mass" to QuantityFixture.oneKilogram, )), @@ -278,7 +278,7 @@ class LcaExpressionReducerTest { inputs = listOf(block), ) val sourceOps = mockk>() - every { sourceOps.readAll(any()) } returns sequenceOf( + every { sourceOps.getAll(any()) } returns sequenceOf( ERecord(mapOf( "mass" to QuantityFixture.oneKilogram, )), @@ -353,7 +353,7 @@ class LcaExpressionReducerTest { inputs = listOf(block), ) val sourceOps = mockk>() - every { sourceOps.readAll(any()) } returns sequenceOf( + every { sourceOps.getAll(any()) } returns sequenceOf( ERecord(emptyMap()), ERecord(emptyMap()), ) From b5b149d7e6ec5b993ae1a25ff45b1311fef9789d Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Tue, 7 May 2024 23:55:34 +0200 Subject: [PATCH 05/26] core: data source config in data source expression/value --- ...cConnectorConfig.kt => ConnectorConfig.kt} | 2 +- .../lcaac/core/config/DataSourceConfig.kt | 9 ++ .../ch/kleis/lcaac/core/config/LcaacConfig.kt | 4 +- .../core/config/LcaacDataSourceConfig.kt | 9 -- .../core/datasource/DataSourceConnector.kt | 7 +- .../core/datasource/DataSourceManager.kt | 17 +-- .../lcaac/core/datasource/csv/CsvConnector.kt | 9 +- .../core/datasource/csv/CsvConnectorConfig.kt | 4 +- .../lcaac/core/lang/evaluator/ToValue.kt | 3 +- .../reducer/DataExpressionReducer.kt | 3 +- .../lang/expression/DataSourceExpression.kt | 5 +- .../lcaac/core/lang/value/DataSourceValue.kt | 5 +- .../lcaac/core/datasource/CsvConnectorTest.kt | 30 ++--- .../core/datasource/DataSourceManagerTest.kt | 26 +++-- .../reducer/DataExpressionReducerTest.kt | 109 ++++++++++++------ .../reducer/LcaExpressionReducerTest.kt | 7 +- .../core/lang/evaluator/step/ReduceTest.kt | 7 +- 17 files changed, 155 insertions(+), 101 deletions(-) rename core/src/main/kotlin/ch/kleis/lcaac/core/config/{LcaacConnectorConfig.kt => ConnectorConfig.kt} (74%) create mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt delete mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacDataSourceConfig.kt diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConnectorConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/ConnectorConfig.kt similarity index 74% rename from core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConnectorConfig.kt rename to core/src/main/kotlin/ch/kleis/lcaac/core/config/ConnectorConfig.kt index 33304ff9..8a415f51 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConnectorConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/ConnectorConfig.kt @@ -1,6 +1,6 @@ package ch.kleis.lcaac.core.config -data class LcaacConnectorConfig( +data class ConnectorConfig( val name: String, val options: Map, ) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt new file mode 100644 index 00000000..97740a54 --- /dev/null +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt @@ -0,0 +1,9 @@ +package ch.kleis.lcaac.core.config + +data class DataSourceConfig( + val name: String, + val connector: String? = null, + val location: String? = null, + val primaryKey: String? = null, + val options: Map = emptyMap(), +) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt index d2006489..c5228573 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt @@ -3,7 +3,7 @@ package ch.kleis.lcaac.core.config data class LcaacConfig( val name: String, val description: String, - val datasources: Map, - val connectors: Map, + val datasources: Map, + val connectors: Map, ) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacDataSourceConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacDataSourceConfig.kt deleted file mode 100644 index 80f5ea6d..00000000 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacDataSourceConfig.kt +++ /dev/null @@ -1,9 +0,0 @@ -package ch.kleis.lcaac.core.config - -data class LcaacDataSourceConfig( - val name: String, - val connector: String = "csv", - val location: String = "$name.csv", - val primaryKey: String = "id", - val options: Map = emptyMap(), -) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceConnector.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceConnector.kt index c12cb725..e6bcd8e8 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceConnector.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceConnector.kt @@ -1,11 +1,10 @@ package ch.kleis.lcaac.core.datasource -import ch.kleis.lcaac.core.config.LcaacConnectorConfig -import ch.kleis.lcaac.core.config.LcaacDataSourceConfig +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.lang.expression.ERecord import ch.kleis.lcaac.core.lang.value.DataSourceValue interface DataSourceConnector { - fun getFirst(config: LcaacDataSourceConfig, source: DataSourceValue): ERecord - fun getAll(config: LcaacDataSourceConfig, source: DataSourceValue): Sequence> + fun getFirst(config: DataSourceConfig, source: DataSourceValue): ERecord + fun getAll(config: DataSourceConfig, source: DataSourceValue): Sequence> } diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt index 63a84c88..1058e4f0 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt @@ -1,7 +1,7 @@ package ch.kleis.lcaac.core.datasource import ch.kleis.lcaac.core.config.LcaacConfig -import ch.kleis.lcaac.core.config.LcaacDataSourceConfig +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer import ch.kleis.lcaac.core.lang.expression.DataExpression import ch.kleis.lcaac.core.lang.expression.EQuantityAdd @@ -22,24 +22,25 @@ class DataSourceManager( connectors[connectorName] = connector } - private fun configOf(name: String): LcaacDataSourceConfig { - return config.datasources[name] - ?: throw IllegalArgumentException("No configuration found for datasource '$name'") + private fun configOf(source: DataSourceValue): DataSourceConfig { +// return config.datasources[name] +// ?: throw IllegalArgumentException("No configuration found for datasource '$name'") + TODO("Implement me") } - private fun connectorOf(config: LcaacDataSourceConfig): DataSourceConnector { + private fun connectorOf(config: DataSourceConfig): DataSourceConnector { return connectors[config.connector] ?: throw IllegalArgumentException("Unknown connect '${config.connector}'") } override fun getFirst(source: DataSourceValue): ERecord { - val sourceConfig = configOf(source.name) + val sourceConfig = configOf(source) val connector = connectorOf(sourceConfig) return connector.getFirst(sourceConfig, source) } override fun getAll(source: DataSourceValue): Sequence> { - val sourceConfig = configOf(source.name) + val sourceConfig = configOf(source) val connector = connectorOf(sourceConfig) return connector.getAll(sourceConfig, source) } @@ -55,7 +56,7 @@ class DataSourceManager( columns.map { column -> record.entries[column] ?: throw IllegalStateException( - "${source.location}: invalid schema: unknown column '$column'" + "${source.config.name}: invalid schema: unknown column '$column'" ) }.reduce { acc, expression -> reducer.reduce(EQuantityMul(acc, expression)) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt index 9fdbf227..e79dee61 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt @@ -1,7 +1,6 @@ package ch.kleis.lcaac.core.datasource.csv -import ch.kleis.lcaac.core.config.LcaacConnectorConfig -import ch.kleis.lcaac.core.config.LcaacDataSourceConfig +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.datasource.DataSourceConnector import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException import ch.kleis.lcaac.core.lang.evaluator.ToValue @@ -52,7 +51,7 @@ class CsvConnector( } } - override fun getAll(config: LcaacDataSourceConfig, source: DataSourceValue): Sequence> { + override fun getAll(config: DataSourceConfig, source: DataSourceValue): Sequence> { val records = load(config.location, source.schema) val filter = source.filter return records @@ -63,7 +62,7 @@ class CsvConnector( val actual = record.entries[it.key] ?.let { with(ToValue(ops)) { it.toValue() } } ?: throw IllegalStateException( - "${source.location}: invalid schema: unknown column '${it.key}'" + "${source.config.name}: invalid schema: unknown column '${it.key}'" ) actual == expected } else throw EvaluatorException("invalid matching condition $it") @@ -71,7 +70,7 @@ class CsvConnector( } } - override fun getFirst(config: LcaacDataSourceConfig, source: DataSourceValue): ERecord { + override fun getFirst(config: DataSourceConfig, source: DataSourceValue): ERecord { return getAll(config, source).firstOrNull() ?: throw EvaluatorException("no record found in '${config.location}' matching ${source.filter}") } diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt index 6174573c..2f449702 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt @@ -1,6 +1,6 @@ package ch.kleis.lcaac.core.datasource.csv -import ch.kleis.lcaac.core.config.LcaacConnectorConfig +import ch.kleis.lcaac.core.config.ConnectorConfig import ch.kleis.lcaac.core.datasource.csv.CsvConnectorConfig.Companion.CSV_CONNECTOR_KEY_DIRECTORY import ch.kleis.lcaac.core.datasource.csv.CsvConnectorConfig.Companion.CSV_CONNECTOR_NAME import java.io.File @@ -15,7 +15,7 @@ data class CsvConnectorConfig( } } -fun LcaacConnectorConfig.csv(): CsvConnectorConfig? { +fun ConnectorConfig.csv(): CsvConnectorConfig? { if (this.name != CSV_CONNECTOR_NAME) { return null } diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/ToValue.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/ToValue.kt index a8fd5b6c..359ee8dd 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/ToValue.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/ToValue.kt @@ -156,8 +156,7 @@ class ToValue( fun DataSourceExpression.toValue(): DataSourceValue { return when(this) { is EDataSource -> DataSourceValue( - this.name, - this.location, + this.config, this.schema.mapValues { it.value.toValue() }, this.filter.mapValues { it.value.toValue() }, ) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducer.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducer.kt index f6b616f0..896ed983 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducer.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducer.kt @@ -63,7 +63,8 @@ class DataExpressionReducer( val invalidKeys = f.keys .filter { s.containsKey(it) } .filter { s[it]!! !is StringExpression } - if (invalidKeys.isNotEmpty()) throw EvaluatorException("data source '${expression.location}': cannot match on numeric column(s) $invalidKeys") + if (invalidKeys.isNotEmpty()) + throw EvaluatorException("data source '${expression.config.name}': cannot match on numeric column(s) $invalidKeys") expression.copy( schema = s, filter = f, diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/expression/DataSourceExpression.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/expression/DataSourceExpression.kt index e96d92f5..3b25be2a 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/expression/DataSourceExpression.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/expression/DataSourceExpression.kt @@ -1,10 +1,11 @@ package ch.kleis.lcaac.core.lang.expression +import ch.kleis.lcaac.core.config.DataSourceConfig + sealed interface DataSourceExpression data class EDataSource ( - val name: String, - @Deprecated("will be removed") val location: String, + val config: DataSourceConfig, val schema: Map>, val filter: Map> = emptyMap(), ) : DataSourceExpression diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataSourceValue.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataSourceValue.kt index ac3fa335..d09d4b2e 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataSourceValue.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataSourceValue.kt @@ -1,8 +1,9 @@ package ch.kleis.lcaac.core.lang.value +import ch.kleis.lcaac.core.config.DataSourceConfig + data class DataSourceValue( - val name: String, - @Deprecated("will be removed") val location: String, + val config: DataSourceConfig, val schema: Map>, val filter: Map> = emptyMap(), ) diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvConnectorTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvConnectorTest.kt index 622303b3..5f9cdeeb 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvConnectorTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/CsvConnectorTest.kt @@ -1,18 +1,14 @@ package ch.kleis.lcaac.core.datasource -import ch.kleis.lcaac.core.config.LcaacDataSourceConfig +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.datasource.csv.CsvConnector import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException -import ch.kleis.lcaac.core.lang.expression.EQuantityScale import ch.kleis.lcaac.core.lang.expression.ERecord import ch.kleis.lcaac.core.lang.expression.EStringLiteral -import ch.kleis.lcaac.core.lang.expression.EUnitLiteral import ch.kleis.lcaac.core.lang.fixture.QuantityFixture import ch.kleis.lcaac.core.lang.fixture.QuantityValueFixture -import ch.kleis.lcaac.core.lang.fixture.UnitFixture import ch.kleis.lcaac.core.lang.value.DataSourceValue import ch.kleis.lcaac.core.lang.value.StringValue -import ch.kleis.lcaac.core.math.basic.BasicNumber import ch.kleis.lcaac.core.math.basic.BasicOperations import io.mockk.mockk import org.junit.jupiter.api.Test @@ -45,8 +41,10 @@ class CsvConnectorTest { mockFileLoader(), ) val source = DataSourceValue( - name = "source", - location = "source.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), schema = mapOf( "geo" to StringValue("FR"), "n_items" to QuantityValueFixture.oneUnit, @@ -56,7 +54,7 @@ class CsvConnectorTest { "geo" to StringValue("FR") ) ) - val config = LcaacDataSourceConfig( + val config = DataSourceConfig( name = "source", location = "source.csv", ) @@ -94,8 +92,10 @@ class CsvConnectorTest { mockFileLoader(), ) val source = DataSourceValue( - name = "source", - location = "source.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), schema = mapOf( "geo" to StringValue("FR"), "n_items" to QuantityValueFixture.oneUnit, @@ -105,7 +105,7 @@ class CsvConnectorTest { "geo" to StringValue("UK") ) ) - val config = LcaacDataSourceConfig( + val config = DataSourceConfig( name = "source", location = "source.csv", ) @@ -131,8 +131,10 @@ class CsvConnectorTest { mockFileLoader(), ) val source = DataSourceValue( - name = "source", - location = "source.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), schema = mapOf( "geo" to StringValue("FR"), "n_items" to QuantityValueFixture.oneUnit, @@ -142,7 +144,7 @@ class CsvConnectorTest { "geo" to StringValue("A_NON_EXISTING_COUNTRY") ) ) - val config = LcaacDataSourceConfig( + val config = DataSourceConfig( name = "source", location = "source.csv", ) diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManagerTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManagerTest.kt index 4a371364..0844652b 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManagerTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManagerTest.kt @@ -1,8 +1,8 @@ package ch.kleis.lcaac.core.datasource import ch.kleis.lcaac.core.config.LcaacConfig -import ch.kleis.lcaac.core.config.LcaacConnectorConfig -import ch.kleis.lcaac.core.config.LcaacDataSourceConfig +import ch.kleis.lcaac.core.config.ConnectorConfig +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.lang.expression.EQuantityScale import ch.kleis.lcaac.core.lang.expression.ERecord import ch.kleis.lcaac.core.lang.expression.EStringLiteral @@ -21,10 +21,10 @@ import kotlin.test.assertEquals class DataSourceManagerTest { private val connectorName = "connector" - private val connectorConfig = LcaacConnectorConfig(name = connectorName, options = emptyMap()) + private val connectorConfig = ConnectorConfig(name = connectorName, options = emptyMap()) private val sourceName = "source" - private val sourceConfig = LcaacDataSourceConfig(name = sourceName, connector = connectorName) + private val sourceConfig = DataSourceConfig(name = sourceName, connector = connectorName) private val config = LcaacConfig( name = "project", @@ -63,8 +63,10 @@ class DataSourceManagerTest { val sourceOps = DataSourceManager(config, ops) sourceOps.registerConnector(connectorName, connector) val source = DataSourceValue( - name = sourceName, - location = "source.csv", + config = DataSourceConfig( + name = sourceName, + location = "source.csv", + ), schema = mapOf( "geo" to StringValue("FR"), "n_items" to QuantityValueFixture.oneUnit, @@ -112,8 +114,10 @@ class DataSourceManagerTest { val sourceOps = DataSourceManager(config, ops) sourceOps.registerConnector(connectorName, connector) val source = DataSourceValue( - name = sourceName, - location = "source.csv", + config = DataSourceConfig( + name = sourceName, + location = "source.csv", + ), schema = mapOf( "geo" to StringValue("FR"), "n_items" to QuantityValueFixture.oneUnit, @@ -161,8 +165,10 @@ class DataSourceManagerTest { val sourceOps = DataSourceManager(config, ops) sourceOps.registerConnector(connectorName, connector) val source = DataSourceValue( - name = sourceName, - location = "source.csv", + config = DataSourceConfig( + name = sourceName, + location = "source.csv", + ), schema = mapOf( "geo" to StringValue("FR"), "n_items" to QuantityValueFixture.oneUnit, diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducerTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducerTest.kt index 45db298f..86dcf9f8 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducerTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducerTest.kt @@ -1,5 +1,6 @@ package ch.kleis.lcaac.core.lang.evaluator.reducer +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.datasource.DataSourceOperations import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.dimension.Dimension @@ -34,8 +35,10 @@ class DataExpressionReducerTest { fun reduceDataSource_withRefInSchema() { // given val dataSource = EDataSource( - name = "source", - location = "source.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), schema = mapOf( "col" to EQuantityScale(BasicNumber(2.0), EDataRef("foo")), ), @@ -52,8 +55,10 @@ class DataExpressionReducerTest { // then val expected = EDataSource( - name = "source", - location = "source.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), schema = mapOf( "col" to QuantityFixture.twoKilograms, ), @@ -65,8 +70,10 @@ class DataExpressionReducerTest { fun reduceDataSource_filterWithRef() { // given val dataSource = EDataSource( - name = "source", - location = "source.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), schema = mapOf( "id" to EStringLiteral("foo"), ), @@ -92,8 +99,10 @@ class DataExpressionReducerTest { fun reduceDataSource_dataSourceRef() { // given val dataSource = EDataSource( - name = "source", - location = "source.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), schema = mapOf( "volume" to QuantityFixture.oneLitre, "mass" to QuantityFixture.oneKilogram, @@ -114,8 +123,10 @@ class DataExpressionReducerTest { fun reduceDataSource_filter() { // given val dataSource = EDataSource( - name = "source", - location = "source.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), schema = mapOf( "label" to EStringLiteral("value"), "volume" to QuantityFixture.oneLitre, @@ -142,12 +153,17 @@ class DataExpressionReducerTest { @Test fun reduceDataSource_filter_accumulation() { // given - val dataSource = EDataSource(name = "source", location = "source.csv", schema = mapOf( - "label" to EStringLiteral("value"), - "geo" to EStringLiteral("FR"), - "volume" to QuantityFixture.oneLitre, - "mass" to QuantityFixture.oneKilogram, - ), filter = mapOf("geo" to EStringLiteral("UK"))) + val dataSource = EDataSource( + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), + schema = mapOf( + "label" to EStringLiteral("value"), + "geo" to EStringLiteral("FR"), + "volume" to QuantityFixture.oneLitre, + "mass" to QuantityFixture.oneKilogram, + ), filter = mapOf("geo" to EStringLiteral("UK"))) val dataSourceRegister = DataSourceRegister.from(mapOf(DataSourceKey("source") to dataSource)) val expression = EFilter(EDataSourceRef("source"), mapOf("label" to EStringLiteral("some_value"))) val reducer = DataExpressionReducer(DataRegister.empty(), dataSourceRegister, ops, sourceOps) @@ -166,17 +182,22 @@ class DataExpressionReducerTest { @Test fun reduceDataSource_filter_whenInvalid() { // given - val dataSource = EDataSource(name = "source", location = "source.csv", schema = mapOf( - "volume" to QuantityFixture.oneLitre, - "mass" to QuantityFixture.oneKilogram, - )) + val dataSource = EDataSource( + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), + schema = mapOf( + "volume" to QuantityFixture.oneLitre, + "mass" to QuantityFixture.oneKilogram, + )) val dataSourceRegister = DataSourceRegister.from(mapOf(DataSourceKey("source") to dataSource)) val expression = EFilter(EDataSourceRef("source"), mapOf("volume" to EStringLiteral("some_value"))) val reducer = DataExpressionReducer(DataRegister.empty(), dataSourceRegister, ops, sourceOps) // when val e = assertThrows { reducer.reduceDataSource(expression) } - assertEquals("data source 'source.csv': cannot match on numeric column(s) [volume]", e.message) + assertEquals("data source 'source': cannot match on numeric column(s) [volume]", e.message) } /* @@ -188,8 +209,10 @@ class DataExpressionReducerTest { // given val expression = ESumProduct(EDataSourceRef("source"), listOf("volume", "mass")) val dataSource = EDataSource( - name = "source", - location = "source.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), schema = mapOf( "volume" to QuantityFixture.oneLitre, "mass" to QuantityFixture.oneKilogram, @@ -226,8 +249,10 @@ class DataExpressionReducerTest { // given val expression = ESumProduct(EDataSourceRef("foo"), listOf("mass")) val dataSource = EDataSource( - name = "source", - location = "source.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), schema = mapOf( "mass" to QuantityFixture.oneKilogram, ), @@ -257,8 +282,10 @@ class DataExpressionReducerTest { @Test fun reduce_whenFirstRecordOfDataSourceRef() { val dataSource = EDataSource( - name = "source", - location = "source.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), schema = mapOf( "mass" to QuantityFixture.oneKilogram ) @@ -291,9 +318,14 @@ class DataExpressionReducerTest { @Test fun reduce_whenDefaultRecordOfDataSourceRef() { // given - val dataSource = EDataSource(name = "source", location = "source.csv", schema = mapOf( - "x" to EDataRef("a"), - )) + val dataSource = EDataSource( + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), + schema = mapOf( + "x" to EDataRef("a"), + )) val record = EDefaultRecordOf(EDataSourceRef("source")) val reducer = DataExpressionReducer( Register.from(mapOf( @@ -319,9 +351,14 @@ class DataExpressionReducerTest { @Test fun reduce_whenDefaultRecordOfDataSourceRef_invalidRef() { // given - val dataSource = EDataSource(name="source", location = "source.csv", schema = mapOf( - "x" to EDataRef("a"), - )) + val dataSource = EDataSource( + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), + schema = mapOf( + "x" to EDataRef("a"), + )) val record = EDefaultRecordOf(EDataSourceRef("foo")) val reducer = DataExpressionReducer( Register.from(mapOf( @@ -980,8 +1017,10 @@ class DataExpressionReducerTest { fun reduce_whenUnitOf_withDataSourceExpression() { // given val dataSource = EDataSource( - name = "source", - location = "source.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), schema = mapOf( "n_items" to EQuantityScale(BasicNumber(0.0), UnitFixture.unit), "mass" to EQuantityScale(BasicNumber(0.0), UnitFixture.kg), diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducerTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducerTest.kt index cee4495c..054a01b4 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducerTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducerTest.kt @@ -1,5 +1,6 @@ package ch.kleis.lcaac.core.lang.evaluator.reducer +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.datasource.DataSourceOperations import ch.kleis.lcaac.core.lang.expression.* import ch.kleis.lcaac.core.lang.fixture.* @@ -16,8 +17,10 @@ class LcaExpressionReducerTest { private val ops = BasicOperations private val sourceOps = mockk>() private val emptyDataSource = EDataSource( - name = "source", - "location.csv", + config = DataSourceConfig( + name = "source", + location = "source.csv", + ), emptyMap(), emptyMap() ) diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/step/ReduceTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/step/ReduceTest.kt index 111ff84b..9294b047 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/step/ReduceTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/step/ReduceTest.kt @@ -1,5 +1,6 @@ package ch.kleis.lcaac.core.lang.evaluator.step +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.datasource.DataSourceOperations import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException @@ -38,8 +39,10 @@ class ReduceTest { ) ) val dataSource = EDataSource( - name = "source", - location = "foo.csv", + config = DataSourceConfig( + name = "source", + location = "foo.csv", + ), schema = mapOf( "mass" to QuantityFixture.oneLitre ) From 9f035907e7a8c059313523d9581e93b037ad1d38 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Wed, 8 May 2024 00:36:36 +0200 Subject: [PATCH 06/26] core: data source config: override mechanism --- .../lcaac/core/config/DataSourceConfig.kt | 22 ++++++++++++++++++- .../core/datasource/DataSourceManager.kt | 10 +++++---- .../lcaac/core/datasource/csv/CsvConnector.kt | 3 ++- .../core/datasource/csv/CsvConnectorConfig.kt | 12 +++++----- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt index 97740a54..25f3ba5d 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt @@ -1,9 +1,29 @@ package ch.kleis.lcaac.core.config +import arrow.typeclasses.Semigroup + data class DataSourceConfig( val name: String, val connector: String? = null, val location: String? = null, val primaryKey: String? = null, val options: Map = emptyMap(), -) +) { + companion object { + fun merger(name: String) = Semigroup { b -> + if (b.name != name) { + throw IllegalArgumentException("Cannot combine config for '$name' with config for '${b.name}'") + } + /* + b overrides a's fields + */ + DataSourceConfig( + name = name, + connector = b.connector ?: this.connector, + location = b.location ?: this.location, + primaryKey = b.primaryKey ?: this.primaryKey, + options = b.options + this.options, + ) + } + } +} diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt index 1058e4f0..3af7542b 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt @@ -1,7 +1,7 @@ package ch.kleis.lcaac.core.datasource -import ch.kleis.lcaac.core.config.LcaacConfig import ch.kleis.lcaac.core.config.DataSourceConfig +import ch.kleis.lcaac.core.config.LcaacConfig import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer import ch.kleis.lcaac.core.lang.expression.DataExpression import ch.kleis.lcaac.core.lang.expression.EQuantityAdd @@ -23,9 +23,11 @@ class DataSourceManager( } private fun configOf(source: DataSourceValue): DataSourceConfig { -// return config.datasources[name] -// ?: throw IllegalArgumentException("No configuration found for datasource '$name'") - TODO("Implement me") + return with(DataSourceConfig.merger(source.config.name)) { + config.datasources[source.config.name] + ?.let { source.config.combine(it) } + ?: source.config + } } private fun connectorOf(config: DataSourceConfig): DataSourceConnector { diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt index e79dee61..8fb8b032 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt @@ -52,7 +52,8 @@ class CsvConnector( } override fun getAll(config: DataSourceConfig, source: DataSourceValue): Sequence> { - val records = load(config.location, source.schema) + val location = config.location ?: throw IllegalArgumentException("Missing location in configuration for datasource '${config.name}'") + val records = load(location, source.schema) val filter = source.filter return records .filter { record -> diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt index 2f449702..93dc82fb 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt @@ -19,10 +19,10 @@ fun ConnectorConfig.csv(): CsvConnectorConfig? { if (this.name != CSV_CONNECTOR_NAME) { return null } - val directory = this.options[CSV_CONNECTOR_KEY_DIRECTORY] - ?.let { Path.of(it).toFile() } - ?: return null - return CsvConnectorConfig( - directory, - ) + val directory = this.options[CSV_CONNECTOR_KEY_DIRECTORY] + ?.let { Path.of(it).toFile() } + ?: return null + return CsvConnectorConfig( + directory, + ) } From 57559bb2933556c981c75fa922a5860a5980a5e2 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Wed, 8 May 2024 08:23:28 +0200 Subject: [PATCH 07/26] grammar: datasource: added fields: primary key, block meta --- grammar/src/main/antlr/LcaLang.g4 | 6 +++++- .../ch/kleis/lcaac/grammar/CoreMapper.kt | 20 ++++++++++++++----- .../ch/kleis/lcaac/grammar/CoreMapperTest.kt | 16 ++++++++++++++- .../ch/kleis/lcaac/grammar/LoaderTest.kt | 13 ++++++++---- 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/grammar/src/main/antlr/LcaLang.g4 b/grammar/src/main/antlr/LcaLang.g4 index 7bfbcc6d..a78d4b80 100644 --- a/grammar/src/main/antlr/LcaLang.g4 +++ b/grammar/src/main/antlr/LcaLang.g4 @@ -35,13 +35,16 @@ globalAssignment dataSourceDefinition : DATASOURCE_KEYWORD dataSourceRef LBRACE ( - locationField | schema + locationField | primaryKeyField | schema | block_meta )* RBRACE ; locationField : LOCATION EQUAL STRING_LITERAL ; +primaryKeyField + : PRIMARY_KEY EQUAL STRING_LITERAL + ; schema : SCHEMA_KEYWORD LBRACE columnDefinition* @@ -364,6 +367,7 @@ LABELS_KEYWORD : 'labels' ; DATASOURCE_KEYWORD : 'datasource' ; LOCATION : 'location' ; +PRIMARY_KEY : 'primary_key' ; SCHEMA_KEYWORD : 'schema' ; TEST_KEYWORD : 'test' ; diff --git a/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt b/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt index 3f236411..d215f026 100644 --- a/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt +++ b/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt @@ -2,6 +2,7 @@ package ch.kleis.lcaac.grammar +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.dimension.Dimension import ch.kleis.lcaac.core.lang.dimension.UnitSymbol @@ -364,18 +365,27 @@ class CoreMapper( fun dataSourceDefinition(ctx: LcaLangParser.DataSourceDefinitionContext): EDataSource { val name = ctx.dataSourceRef().uid().ID().innerText() - val locationField = ctx.locationField().firstOrNull() - ?: throw LoaderException("missing location field in datasource $name") - val location = locationField.STRING_LITERAL().innerText() + val location = ctx.locationField().firstOrNull()?.STRING_LITERAL()?.innerText() + val primaryKey = ctx.primaryKeyField().firstOrNull()?.STRING_LITERAL()?.innerText() val schemaBlock = ctx.schema().firstOrNull() ?: throw LoaderException("missing schema in datasource $name") val schema = schemaBlock.columnDefinition().associate { column -> val key = column.columnRef().innerText() val value = dataExpression(column.dataExpression()) key to value } + val options = ctx.block_meta().flatMap { it.meta_assignment() } + .associate { assignment -> + val key = assignment.STRING_LITERAL(0).innerText() + val value = assignment.STRING_LITERAL(1).innerText() + key to value + } return EDataSource( - name = name, - location = location, + config = DataSourceConfig( + name = name, + location = location, + primaryKey = primaryKey, + options = options, + ), schema = schema, ) } diff --git a/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt b/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt index 8caad2a3..5fe89cbb 100644 --- a/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt +++ b/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt @@ -1,5 +1,6 @@ package ch.kleis.lcaac.grammar +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.expression.* import ch.kleis.lcaac.core.math.basic.BasicNumber @@ -195,10 +196,14 @@ class CoreMapperTest { val ctx = LcaLangFixture.parser(""" datasource source { location = "file.csv" + primary_key = "geo" schema { mass = 1 kg geo = "FR" } + meta { + "description": "This is a description" + } } """.trimIndent()).dataSourceDefinition() val mapper = CoreMapper(ops) @@ -207,7 +212,16 @@ class CoreMapperTest { val actual = mapper.dataSourceDefinition(ctx) // then - val expected = EDataSource(name = "source", location = "file.csv", schema = mapOf( + val expected = EDataSource( + config = DataSourceConfig( + name = "source", + location = "file.csv", + primaryKey = "geo", + options = mapOf( + "description" to "This is a description", + ) + ), + schema = mapOf( "mass" to EQuantityScale(BasicNumber(1.0), EDataRef("kg")), "geo" to EStringLiteral("FR"), )) diff --git a/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/LoaderTest.kt b/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/LoaderTest.kt index 6e299ff0..683dba89 100644 --- a/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/LoaderTest.kt +++ b/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/LoaderTest.kt @@ -1,5 +1,6 @@ package ch.kleis.lcaac.grammar +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.dimension.Dimension import ch.kleis.lcaac.core.lang.dimension.UnitSymbol @@ -75,8 +76,10 @@ class LoaderTest { // then val expected = EDataSource( - name = "source", - location = "file.csv", + config = DataSourceConfig( + name = "source", + location = "file.csv", + ), schema = mapOf( "mass" to EQuantityScale(BasicNumber(1.0), EDataRef("kg")), ) @@ -106,8 +109,10 @@ class LoaderTest { // then val expected = EDataSource( - name = "source", - location = "file.csv", + config = DataSourceConfig( + name = "source", + location = "file.csv", + ), schema = mapOf( "mass" to EQuantityScale(BasicNumber(1.0), EDataRef("kg")), "geo" to EStringLiteral("FR"), From 449455bae5dd3a91676e12cf6ae70f103cc2058e Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Wed, 8 May 2024 13:27:01 +0200 Subject: [PATCH 08/26] core: serializable lcaac config --- .../main/kotlin/ch/kleis/lcaac/core/config/ConnectorConfig.kt | 3 +++ .../main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt | 2 ++ core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt | 3 +++ 3 files changed, 8 insertions(+) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/ConnectorConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/ConnectorConfig.kt index 8a415f51..09b08dde 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/config/ConnectorConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/ConnectorConfig.kt @@ -1,5 +1,8 @@ package ch.kleis.lcaac.core.config +import kotlinx.serialization.Serializable + +@Serializable data class ConnectorConfig( val name: String, val options: Map, diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt index 25f3ba5d..8510d9ce 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt @@ -1,7 +1,9 @@ package ch.kleis.lcaac.core.config import arrow.typeclasses.Semigroup +import kotlinx.serialization.Serializable +@Serializable data class DataSourceConfig( val name: String, val connector: String? = null, diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt index c5228573..728b8e8a 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt @@ -1,5 +1,8 @@ package ch.kleis.lcaac.core.config +import kotlinx.serialization.Serializable + +@Serializable data class LcaacConfig( val name: String, val description: String, From 5d0841ea8b1b02d0495da2402de4b2e11f5cea52 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Wed, 8 May 2024 15:09:19 +0200 Subject: [PATCH 09/26] core: refactor connector factory, default data source ops --- .../lcaac/core/config/DataSourceConfig.kt | 20 ++++++++++-- .../ch/kleis/lcaac/core/config/LcaacConfig.kt | 21 +++++++++--- .../lcaac/core/datasource/ConnectorFactory.kt | 20 ++++++++++++ .../core/datasource/DataSourceConnector.kt | 1 + ...ager.kt => DefaultDataSourceOperations.kt} | 15 ++++----- .../lcaac/core/datasource/csv/CsvConnector.kt | 5 +++ .../core/datasource/csv/CsvConnectorConfig.kt | 2 +- ....kt => DefaultDataSourceOperationsTest.kt} | 32 +++++++++++-------- 8 files changed, 85 insertions(+), 31 deletions(-) create mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorFactory.kt rename core/src/main/kotlin/ch/kleis/lcaac/core/datasource/{DataSourceManager.kt => DefaultDataSourceOperations.kt} (85%) rename core/src/test/kotlin/ch/kleis/lcaac/core/datasource/{DataSourceManagerTest.kt => DefaultDataSourceOperationsTest.kt} (85%) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt index 8510d9ce..b90032f3 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/DataSourceConfig.kt @@ -1,17 +1,31 @@ package ch.kleis.lcaac.core.config import arrow.typeclasses.Semigroup +import ch.kleis.lcaac.core.datasource.csv.CsvConnectorConfig import kotlinx.serialization.Serializable @Serializable data class DataSourceConfig( val name: String, - val connector: String? = null, - val location: String? = null, - val primaryKey: String? = null, + val connector: String? = CsvConnectorConfig.CSV_CONNECTOR_NAME, + val location: String? = "$name.csv", + val primaryKey: String? = "id", val options: Map = emptyMap(), ) { companion object { + fun completeWithDefaults(config: DataSourceConfig): DataSourceConfig { + val defaultConfig = DataSourceConfig( + name = config.name, + connector = CsvConnectorConfig.CSV_CONNECTOR_NAME, + location = "${config.name}.csv", + primaryKey = "id", + options = emptyMap(), + ) + return with(merger(config.name)) { + defaultConfig.combine(config) + } + } + fun merger(name: String) = Semigroup { b -> if (b.name != name) { throw IllegalArgumentException("Cannot combine config for '$name' with config for '${b.name}'") diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt index 728b8e8a..838a331e 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt @@ -4,9 +4,20 @@ import kotlinx.serialization.Serializable @Serializable data class LcaacConfig( - val name: String, - val description: String, - val datasources: Map, - val connectors: Map, -) + val name: String = "", + val description: String = "", + val datasources: List = emptyList(), + val connectors: List = emptyList(), +) { + private val datasourcesMap = datasources.associateBy { it.name } + private val connectorsMap = connectors.associateBy { it.name } + + fun getDataSource(name: String): DataSourceConfig? { + return datasourcesMap[name] + } + + fun getConnector(name: String): ConnectorConfig? { + return connectorsMap[name] + } +} diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorFactory.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorFactory.kt new file mode 100644 index 00000000..6f9c7d0e --- /dev/null +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorFactory.kt @@ -0,0 +1,20 @@ +package ch.kleis.lcaac.core.datasource + +import ch.kleis.lcaac.core.config.ConnectorConfig +import ch.kleis.lcaac.core.datasource.csv.CsvConnector +import ch.kleis.lcaac.core.datasource.csv.csv +import ch.kleis.lcaac.core.math.QuantityOperations + +interface ConnectorFactory { + fun buildOrNull(config: ConnectorConfig): DataSourceConnector? + + companion object { + fun default(ops: QuantityOperations) = object : ConnectorFactory { + override fun buildOrNull(config: ConnectorConfig): DataSourceConnector? { + return config + .csv()?.let { CsvConnector(it, ops) } + // add other cases here + } + } + } +} diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceConnector.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceConnector.kt index e6bcd8e8..438dac27 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceConnector.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceConnector.kt @@ -5,6 +5,7 @@ import ch.kleis.lcaac.core.lang.expression.ERecord import ch.kleis.lcaac.core.lang.value.DataSourceValue interface DataSourceConnector { + fun getName(): String fun getFirst(config: DataSourceConfig, source: DataSourceValue): ERecord fun getAll(config: DataSourceConfig, source: DataSourceValue): Sequence> } diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt similarity index 85% rename from core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt rename to core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt index 3af7542b..87548653 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManager.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt @@ -12,19 +12,18 @@ import ch.kleis.lcaac.core.lang.value.DataSourceValue import ch.kleis.lcaac.core.math.QuantityOperations import ch.kleis.lcaac.core.prelude.Prelude -class DataSourceManager( +class DefaultDataSourceOperations( private val config: LcaacConfig, private val ops: QuantityOperations, + private val connectorFactory: ConnectorFactory = ConnectorFactory.default(ops) ) : DataSourceOperations { - private val connectors = HashMap>() - - fun registerConnector(connectorName: String, connector: DataSourceConnector) { - connectors[connectorName] = connector - } + private val connectors = config.connectors + .mapNotNull { connectorFactory.buildOrNull(it) } + .associateBy { it.getName() } private fun configOf(source: DataSourceValue): DataSourceConfig { return with(DataSourceConfig.merger(source.config.name)) { - config.datasources[source.config.name] + config.getDataSource(source.config.name) ?.let { source.config.combine(it) } ?: source.config } @@ -32,7 +31,7 @@ class DataSourceManager( private fun connectorOf(config: DataSourceConfig): DataSourceConnector { return connectors[config.connector] - ?: throw IllegalArgumentException("Unknown connect '${config.connector}'") + ?: throw IllegalArgumentException("Unknown connector '${config.connector}'") } override fun getFirst(source: DataSourceValue): ERecord { diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt index 8fb8b032..48ce13e8 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt @@ -27,6 +27,7 @@ class CsvConnector( csvFile.toFile().inputStream() } ) : DataSourceConnector { + private fun load(location: String, schema: Map>): Sequence> { val inputStream = fileLoader(location) val parser = CSVParser(inputStream.reader(), format) @@ -75,6 +76,10 @@ class CsvConnector( return getAll(config, source).firstOrNull() ?: throw EvaluatorException("no record found in '${config.location}' matching ${source.filter}") } + + override fun getName(): String { + return CsvConnectorConfig.CSV_CONNECTOR_NAME + } } private val format = CSVFormat.DEFAULT.builder() diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt index 93dc82fb..a1815889 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt @@ -21,7 +21,7 @@ fun ConnectorConfig.csv(): CsvConnectorConfig? { } val directory = this.options[CSV_CONNECTOR_KEY_DIRECTORY] ?.let { Path.of(it).toFile() } - ?: return null + ?: Path.of(".").toFile() return CsvConnectorConfig( directory, ) diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManagerTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt similarity index 85% rename from core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManagerTest.kt rename to core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt index 0844652b..180d268d 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DataSourceManagerTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt @@ -19,7 +19,7 @@ import io.mockk.mockk import kotlin.test.Test import kotlin.test.assertEquals -class DataSourceManagerTest { +class DefaultDataSourceOperationsTest { private val connectorName = "connector" private val connectorConfig = ConnectorConfig(name = connectorName, options = emptyMap()) @@ -29,12 +29,8 @@ class DataSourceManagerTest { private val config = LcaacConfig( name = "project", description = "description", - datasources = mapOf( - sourceName to sourceConfig - ), - connectors = mapOf( - connectorName to connectorConfig - ), + datasources = listOf(sourceConfig), + connectors = listOf(connectorConfig), ) private val ops = BasicOperations @@ -42,6 +38,7 @@ class DataSourceManagerTest { fun getAll() { // given val connector = mockk>() + every { connector.getName() } returns connectorName every { connector.getAll(any(), any()) } returns sequenceOf( ERecord(mapOf( "geo" to EStringLiteral("FR"), @@ -59,9 +56,10 @@ class DataSourceManagerTest { "mass" to QuantityFixture.twoKilograms, )), ) - - val sourceOps = DataSourceManager(config, ops) - sourceOps.registerConnector(connectorName, connector) + val factory = object : ConnectorFactory { + override fun buildOrNull(config: ConnectorConfig): DataSourceConnector = connector + } + val sourceOps = DefaultDataSourceOperations(config, ops, factory) val source = DataSourceValue( config = DataSourceConfig( name = sourceName, @@ -105,14 +103,17 @@ class DataSourceManagerTest { fun getFirst() { // given val connector = mockk>() + every { connector.getName() } returns connectorName every { connector.getFirst(any(), any()) } returns ERecord(mapOf( "geo" to EStringLiteral("FR"), "n_items" to QuantityFixture.oneUnit, "mass" to QuantityFixture.oneKilogram, )) - val sourceOps = DataSourceManager(config, ops) - sourceOps.registerConnector(connectorName, connector) + val factory = object : ConnectorFactory { + override fun buildOrNull(config: ConnectorConfig): DataSourceConnector = connector + } + val sourceOps = DefaultDataSourceOperations(config, ops, factory) val source = DataSourceValue( config = DataSourceConfig( name = sourceName, @@ -144,6 +145,7 @@ class DataSourceManagerTest { fun sumProduct() { // given val connector = mockk>() + every { connector.getName() } returns connectorName every { connector.getAll(any(), any()) } returns sequenceOf( ERecord(mapOf( "geo" to EStringLiteral("FR"), @@ -162,8 +164,10 @@ class DataSourceManagerTest { )), ) - val sourceOps = DataSourceManager(config, ops) - sourceOps.registerConnector(connectorName, connector) + val factory = object : ConnectorFactory { + override fun buildOrNull(config: ConnectorConfig): DataSourceConnector = connector + } + val sourceOps = DefaultDataSourceOperations(config, ops, factory) val source = DataSourceValue( config = DataSourceConfig( name = sourceName, From 2a357d5a05600f72c4569d342c7b7c134106b044 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Wed, 8 May 2024 15:09:42 +0200 Subject: [PATCH 10/26] grammar: when parsing data source def, complete with defaults --- .../kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt b/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt index d215f026..cf955d05 100644 --- a/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt +++ b/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt @@ -234,13 +234,16 @@ class CoreMapper( val dataSource = dataSource(ctx.dataSourceExpression()) EFirstRecordOf(dataSource) } + ctx.DEFAULT_RECORD()?.innerText() -> { val dataSource = dataSource(ctx.dataSourceExpression()) EDefaultRecordOf(dataSource) } + else -> throw IllegalStateException("parsing error: invalid primitive '${ctx.op.text}'") } } + is LcaLangParser.ColGroupContext -> { when (ctx.op.text) { ctx.SUM().innerText() -> { @@ -323,7 +326,7 @@ class CoreMapper( fun LcaLangParser.ColumnRefContext.innerText(): String { return this.uid().innerText() } - + fun LcaLangParser.UidContext.innerText(): String { return this.ID()?.innerText() ?: throw LoaderException("parsing error: invalid uid: ${this.text}") @@ -380,11 +383,13 @@ class CoreMapper( key to value } return EDataSource( - config = DataSourceConfig( - name = name, - location = location, - primaryKey = primaryKey, - options = options, + config = DataSourceConfig.completeWithDefaults( + DataSourceConfig( + name = name, + location = location, + primaryKey = primaryKey, + options = options, + ) ), schema = schema, ) From 54f9f9b372d971713ded498c304a1172985ea09c Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Wed, 8 May 2024 15:10:12 +0200 Subject: [PATCH 11/26] tutorials: 03/02-cff: refactor with lcaac.yaml --- .../02-circular-footprint-formula/00-datasources.lca | 10 ---------- .../02-circular-footprint-formula/lcaac.yaml | 6 ++++++ 2 files changed, 6 insertions(+), 10 deletions(-) create mode 100644 tutorials/03-advanced/02-circular-footprint-formula/lcaac.yaml diff --git a/tutorials/03-advanced/02-circular-footprint-formula/00-datasources.lca b/tutorials/03-advanced/02-circular-footprint-formula/00-datasources.lca index fa8cdaef..4a2dd549 100644 --- a/tutorials/03-advanced/02-circular-footprint-formula/00-datasources.lca +++ b/tutorials/03-advanced/02-circular-footprint-formula/00-datasources.lca @@ -3,7 +3,6 @@ */ datasource material_params { - location = "data/material_params.csv" schema { id = "material-01" A = 1.0 u @@ -32,7 +31,6 @@ datasource material_params { */ datasource R2_data { - location = "data/R2_data.csv" schema { id = "glass-01" geo = "GLO" @@ -47,7 +45,6 @@ datasource R2_data { // VIRGIN datasource Ev_data { - location = "data/Ev_data.csv" schema { id = "material-01" GWP = 1 kg_CO2_Eq @@ -71,7 +68,6 @@ process virgin_production_process { // RECYCLED datasource Erecycled_data { - location = "data/Erecycled_data.csv" schema { id = "material-01" GWP = 1 kg_CO2_Eq @@ -95,7 +91,6 @@ process upstream_recycling_process { // RECYCLING END OF LIFE datasource ErecyclingEol_data { - location = "data/ErecyclingEol_data.csv" schema { id = "material-01" GWP = 1 kg_CO2_Eq @@ -119,7 +114,6 @@ process downstream_recycling_process { // SUBSTITUTE FOR VIRGIN datasource EstarV_data { - location = "data/EstarV_data.csv" schema { id = "material-01" GWP = 1 kg_CO2_Eq @@ -143,7 +137,6 @@ process substitute_production_process { // ENERGY RECOVERY datasource Eer_data { - location = "data/Eer_data.csv" schema { id = "material-01" GWP = 1 kg_CO2_Eq @@ -167,7 +160,6 @@ process energy_recovery_process { // HEAT PRODUCTION datasource Ese_heat_data { - location = "data/Ese_heat_data.csv" schema { geo = "GLO" GWP = 1 kg_CO2_Eq @@ -191,7 +183,6 @@ process heat_production_process { // ELECTRICITY PRODUCTION datasource Ese_elec_data { - location = "data/Ese_elec_data.csv" schema { geo = "GLO" GWP = 1 kg_CO2_Eq @@ -215,7 +206,6 @@ process elec_production_process { // DISPOSAL datasource Ed_data { - location = "data/Ed_data.csv" schema { id = "material-01" GWP = 1 kg_CO2_Eq diff --git a/tutorials/03-advanced/02-circular-footprint-formula/lcaac.yaml b/tutorials/03-advanced/02-circular-footprint-formula/lcaac.yaml new file mode 100644 index 00000000..571a0ba0 --- /dev/null +++ b/tutorials/03-advanced/02-circular-footprint-formula/lcaac.yaml @@ -0,0 +1,6 @@ +name: Circular footprint formula +description: A sample project to illustrate the implementation of the circular footprint formula. +connectors: + - name: csv + options: + directory: "data" From 1fb4aeda0261f6f8bb01454d2690df10423ee3d3 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Wed, 8 May 2024 15:25:43 +0200 Subject: [PATCH 12/26] cli: read lcaac config --- cli/build.gradle.kts | 2 ++ .../ch/kleis/lcaac/cli/cmd/AssessCommand.kt | 24 +++++++++----- .../ch/kleis/lcaac/cli/cmd/LcaacCommand.kt | 3 +- .../ch/kleis/lcaac/cli/cmd/TestCommand.kt | 31 +++++++++++++------ .../ch/kleis/lcaac/cli/cmd/TraceCommand.kt | 26 ++++++++++------ .../kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt | 3 +- .../ch/kleis/lcaac/cli/csv/CsvProcessor.kt | 8 ++--- .../kleis/lcaac/cli/csv/CsvProcessorTest.kt | 15 +++++---- 8 files changed, 70 insertions(+), 42 deletions(-) diff --git a/cli/build.gradle.kts b/cli/build.gradle.kts index 6a549d2e..df36fb83 100644 --- a/cli/build.gradle.kts +++ b/cli/build.gradle.kts @@ -43,6 +43,8 @@ dependencies { implementation("com.github.ajalt.clikt:clikt:4.2.2") implementation("org.apache.commons:commons-csv:1.10.0") + + implementation("com.charleskorn.kaml:kaml:0.59.0") } tasks.test { diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt index 63bed036..476eb0af 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt @@ -4,15 +4,22 @@ import ch.kleis.lcaac.cli.csv.CsvProcessor import ch.kleis.lcaac.cli.csv.CsvRequest import ch.kleis.lcaac.cli.csv.CsvRequestReader import ch.kleis.lcaac.cli.csv.CsvResultWriter +import ch.kleis.lcaac.core.config.LcaacConfig import ch.kleis.lcaac.core.math.basic.BasicOperations import ch.kleis.lcaac.grammar.Loader import ch.kleis.lcaac.grammar.LoaderOption +import com.charleskorn.kaml.Yaml +import com.charleskorn.kaml.decodeFromStream import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.arguments.help -import com.github.ajalt.clikt.parameters.options.* +import com.github.ajalt.clikt.parameters.options.associate +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.help +import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.file import java.io.File +import java.nio.file.Path @Suppress("MemberVisibilityCanBePrivate") class AssessCommand : CliktCommand(name = "assess", help = "Returns the unitary impacts of a process in CSV format") { @@ -24,11 +31,9 @@ class AssessCommand : CliktCommand(name = "assess", help = "Returns the unitary Example: lcaac assess -l model="ABC" -l geo="FR". """.trimIndent()) .associate() - private val getPath = option("-p", "--path").file(canBeFile = false).default(File(".")).help("Path to root folder.") - val path: File by getPath - val dataSourcePath: File by option("--data-path").file(canBeFile = false) - .defaultLazy { getPath.value } - .help("Path to data folder. Default to root folder.") + private val getConfigPath = option("-c", "--config").file().default(File("lcaac.yaml")).help("Configuration file.") + val configFile: File by getConfigPath + val file: File? by option("-f", "--file").file(canBeDir = false) .help(""" CSV file with parameter values. @@ -43,9 +48,12 @@ class AssessCommand : CliktCommand(name = "assess", help = "Returns the unitary .associate() override fun run() { - val files = lcaFiles(path) + val config = configFile.inputStream().use { + Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) + } + val files = lcaFiles(Path.of(".").toFile()) val symbolTable = Loader(BasicOperations).load(files, listOf(LoaderOption.WITH_PRELUDE)) - val processor = CsvProcessor(dataSourcePath, symbolTable) + val processor = CsvProcessor(config, symbolTable) val iterator = loadRequests() val writer = CsvResultWriter() var first = true diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/LcaacCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/LcaacCommand.kt index 498bdc9e..db4acc32 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/LcaacCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/LcaacCommand.kt @@ -3,6 +3,5 @@ package ch.kleis.lcaac.cli.cmd import com.github.ajalt.clikt.core.CliktCommand class LcaacCommand : CliktCommand(name = "lcaac") { - override fun run() { - } + override fun run() {} } diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt index 6693e592..8d86c3b6 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt @@ -1,6 +1,7 @@ package ch.kleis.lcaac.cli.cmd -import ch.kleis.lcaac.core.datasource.CsvSourceOperations +import ch.kleis.lcaac.core.config.LcaacConfig +import ch.kleis.lcaac.core.datasource.DefaultDataSourceOperations import ch.kleis.lcaac.core.math.basic.BasicOperations import ch.kleis.lcaac.core.testing.BasicTestRunner import ch.kleis.lcaac.core.testing.GenericFailure @@ -10,31 +11,40 @@ import ch.kleis.lcaac.grammar.CoreTestMapper import ch.kleis.lcaac.grammar.Loader import ch.kleis.lcaac.grammar.LoaderOption import ch.kleis.lcaac.grammar.parser.LcaLangParser +import com.charleskorn.kaml.Yaml +import com.charleskorn.kaml.decodeFromStream import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.ProgramResult -import com.github.ajalt.clikt.parameters.options.* +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.help +import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.file import java.io.File +import java.nio.file.Path private const val greenTick = "\u2705" private const val redCross = "\u274C" class TestCommand : CliktCommand(name = "test", help = "Run specified tests") { - private val getPath = option("-p", "--path").file(canBeFile = false).default(File(".")).help("Path to root folder.") - val path: File by getPath - val dataSourcePath: File by option("--data-path").file(canBeFile = false) - .defaultLazy { getPath.value } - .help("Path to data folder. Default to root folder.") + private val getConfigPath = option("-c", "--config").file().default(File("lcaac.yaml")).help("Configuration file.") + val configFile: File by getConfigPath val file: File? by option("-f", "--file").file(canBeDir = false) - .help(""" + .help(""" CSV file with parameter values. Example: `lcaac assess -f params.csv`. """.trimIndent()) val showSuccess: Boolean by option("--show-success").flag(default = false).help("Show successful assertions") override fun run() { - val files = lcaFiles(path) + val config = configFile.inputStream().use { + Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) + } + val ops = BasicOperations + val sourceOps = DefaultDataSourceOperations(config, ops) + + val files = lcaFiles(Path.of(".").toFile()) val symbolTable = Loader(ops).load(files, listOf(LoaderOption.WITH_PRELUDE)) val mapper = CoreTestMapper() val cases = files @@ -42,7 +52,8 @@ class TestCommand : CliktCommand(name = "test", help = "Run specified tests") { .map { mapper.test(it) } val runner = BasicTestRunner( symbolTable, - CsvSourceOperations(dataSourcePath, ops)) + sourceOps, + ) val results = cases.map { runner.run(it) } results.forEach { result -> diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt index ab0f0e3e..a558d7bc 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt @@ -1,7 +1,8 @@ package ch.kleis.lcaac.cli.cmd import ch.kleis.lcaac.core.assessment.ContributionAnalysisProgram -import ch.kleis.lcaac.core.datasource.CsvSourceOperations +import ch.kleis.lcaac.core.config.LcaacConfig +import ch.kleis.lcaac.core.datasource.DefaultDataSourceOperations import ch.kleis.lcaac.core.lang.evaluator.Evaluator import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer @@ -14,14 +15,20 @@ import ch.kleis.lcaac.core.math.basic.BasicOperations.toDouble import ch.kleis.lcaac.core.prelude.Prelude.Companion.sanitize import ch.kleis.lcaac.grammar.Loader import ch.kleis.lcaac.grammar.LoaderOption +import com.charleskorn.kaml.Yaml +import com.charleskorn.kaml.decodeFromStream import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.arguments.help -import com.github.ajalt.clikt.parameters.options.* +import com.github.ajalt.clikt.parameters.options.associate +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.help +import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.file import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVPrinter import java.io.File +import java.nio.file.Path @Suppress("MemberVisibilityCanBePrivate") class TraceCommand : CliktCommand(name = "trace", help = "Trace the contributions") { @@ -33,11 +40,8 @@ class TraceCommand : CliktCommand(name = "trace", help = "Trace the contribution Example: lcaac assess -l model="ABC" -l geo="FR". """.trimIndent()) .associate() - private val getPath = option("-p", "--path").file(canBeFile = false).default(File(".")).help("Path to root folder.") - val path: File by getPath - val dataSourcePath: File by option("--data-path").file(canBeFile = false) - .defaultLazy { getPath.value } - .help("Path to data folder. Default to root folder.") + private val getConfigPath = option("-c", "--config").file().default(File("lcaac.yaml")).help("Configuration file.") + val configFile: File by getConfigPath val arguments: Map by option("-D", "--parameter") .help( """ @@ -47,9 +51,13 @@ class TraceCommand : CliktCommand(name = "trace", help = "Trace the contribution .associate() override fun run() { + val config = configFile.inputStream().use { + Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) + } val ops = BasicOperations - val sourceOps = CsvSourceOperations(dataSourcePath, ops) - val files = lcaFiles(path) + val sourceOps = DefaultDataSourceOperations(config, ops) + + val files = lcaFiles(Path.of(".").toFile()) val symbolTable = Loader(ops).load(files, listOf(LoaderOption.WITH_PRELUDE)) val evaluator = Evaluator(symbolTable, ops, sourceOps) val template = symbolTable.getTemplate(name, labels) diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt index d202e773..90d037bc 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt @@ -84,7 +84,8 @@ fun prepareArguments( EStringLiteral(it) } ?: defaultValue.toEStringLiteral() - else -> throw EvaluatorException("datasource '${dataSource.location}': column '${schemaEntry.key}': invalid default value") + else -> throw EvaluatorException("datasource '${dataSource.config.name}': column '${schemaEntry + .key}': invalid default value") } } ERecord(entries) diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessor.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessor.kt index cd4f5e50..c56942be 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessor.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessor.kt @@ -2,21 +2,21 @@ package ch.kleis.lcaac.cli.csv import ch.kleis.lcaac.cli.cmd.prepareArguments import ch.kleis.lcaac.core.assessment.ContributionAnalysisProgram -import ch.kleis.lcaac.core.datasource.CsvSourceOperations +import ch.kleis.lcaac.core.config.LcaacConfig +import ch.kleis.lcaac.core.datasource.DefaultDataSourceOperations import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.evaluator.Evaluator import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer import ch.kleis.lcaac.core.math.basic.BasicNumber import ch.kleis.lcaac.core.math.basic.BasicOperations -import java.io.File class CsvProcessor( - path: File, + config: LcaacConfig, private val symbolTable: SymbolTable, ) { private val ops = BasicOperations - private val sourceOps = CsvSourceOperations(path, ops) + private val sourceOps = DefaultDataSourceOperations(config, ops) private val dataReducer = DataExpressionReducer(symbolTable.data, symbolTable.dataSources, ops, sourceOps) private val evaluator = Evaluator(symbolTable, ops, sourceOps) diff --git a/cli/src/test/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessorTest.kt b/cli/src/test/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessorTest.kt index 454c0cf2..166c23e1 100644 --- a/cli/src/test/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessorTest.kt +++ b/cli/src/test/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessorTest.kt @@ -1,5 +1,6 @@ package ch.kleis.lcaac.cli.csv +import ch.kleis.lcaac.core.config.LcaacConfig import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.dimension.Dimension import ch.kleis.lcaac.core.lang.dimension.UnitSymbol @@ -10,12 +11,10 @@ import ch.kleis.lcaac.grammar.Loader import ch.kleis.lcaac.grammar.LoaderOption import ch.kleis.lcaac.grammar.parser.LcaLangLexer import ch.kleis.lcaac.grammar.parser.LcaLangParser -import io.mockk.mockk import org.antlr.v4.runtime.CharStreams import org.antlr.v4.runtime.CommonTokenStream import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test -import java.io.File class CsvProcessorTest { @Test @@ -42,8 +41,8 @@ class CsvProcessorTest { } """.trimIndent() val symbolTable = load(content) - val path = mockk() - val processor = CsvProcessor(path, symbolTable) + val config = LcaacConfig() + val processor = CsvProcessor(config, symbolTable) val request = CsvRequest( "main", emptyMap(), @@ -106,8 +105,8 @@ class CsvProcessorTest { } """.trimIndent() val symbolTable = load(content) - val path = mockk() - val processor = CsvProcessor(path, symbolTable) + val config = LcaacConfig() + val processor = CsvProcessor(config, symbolTable) val request = CsvRequest( "main", emptyMap(), @@ -162,8 +161,8 @@ class CsvProcessorTest { } """.trimIndent() val symbolTable = load(content) - val path = mockk() - val processor = CsvProcessor(path, symbolTable) + val config = LcaacConfig() + val processor = CsvProcessor(config, symbolTable) val request = CsvRequest( "main", emptyMap(), From fe1aee1559df7a924146657c7842d9d711bdde92 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Wed, 8 May 2024 22:31:39 +0200 Subject: [PATCH 13/26] core: refactor connector factory --- .../lcaac/core/datasource/ConnectorBuilder.kt | 10 +++++++ .../lcaac/core/datasource/ConnectorFactory.kt | 29 +++++++++++-------- .../datasource/DefaultDataSourceOperations.kt | 2 +- .../lcaac/core/datasource/csv/CsvConnector.kt | 6 +++- .../datasource/csv/CsvConnectorBuilder.kt | 13 +++++++++ .../DefaultDataSourceOperationsTest.kt | 18 ++++++------ 6 files changed, 55 insertions(+), 23 deletions(-) create mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorBuilder.kt create mode 100644 core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorBuilder.kt diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorBuilder.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorBuilder.kt new file mode 100644 index 00000000..a74af90e --- /dev/null +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorBuilder.kt @@ -0,0 +1,10 @@ +package ch.kleis.lcaac.core.datasource + +import ch.kleis.lcaac.core.config.ConnectorConfig + +interface ConnectorBuilder { + fun buildOrNull( + factory: ConnectorFactory, + config: ConnectorConfig, + ): DataSourceConnector? +} diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorFactory.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorFactory.kt index 6f9c7d0e..aa017658 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorFactory.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorFactory.kt @@ -1,20 +1,25 @@ package ch.kleis.lcaac.core.datasource import ch.kleis.lcaac.core.config.ConnectorConfig -import ch.kleis.lcaac.core.datasource.csv.CsvConnector -import ch.kleis.lcaac.core.datasource.csv.csv +import ch.kleis.lcaac.core.config.LcaacConfig +import ch.kleis.lcaac.core.datasource.csv.CsvConnectorBuilder import ch.kleis.lcaac.core.math.QuantityOperations -interface ConnectorFactory { - fun buildOrNull(config: ConnectorConfig): DataSourceConnector? +class ConnectorFactory( + private val lcaacConfig: LcaacConfig, + private val ops: QuantityOperations, + builders: List> = listOf(CsvConnectorBuilder()) +) { + private val builders = ArrayList>(builders) - companion object { - fun default(ops: QuantityOperations) = object : ConnectorFactory { - override fun buildOrNull(config: ConnectorConfig): DataSourceConnector? { - return config - .csv()?.let { CsvConnector(it, ops) } - // add other cases here - } - } + fun getLcaacConfig(): LcaacConfig = lcaacConfig + fun getQuantityOperations(): QuantityOperations = ops + + fun register(builder: ConnectorBuilder) { + builders.add(builder) + } + + fun buildOrNull(config: ConnectorConfig): DataSourceConnector? { + return builders.firstNotNullOfOrNull { it.buildOrNull(this, config) } } } diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt index 87548653..53ed0fa3 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt @@ -15,7 +15,7 @@ import ch.kleis.lcaac.core.prelude.Prelude class DefaultDataSourceOperations( private val config: LcaacConfig, private val ops: QuantityOperations, - private val connectorFactory: ConnectorFactory = ConnectorFactory.default(ops) + private val connectorFactory: ConnectorFactory = ConnectorFactory(config, ops) ) : DataSourceOperations { private val connectors = config.connectors .mapNotNull { connectorFactory.buildOrNull(it) } diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt index 48ce13e8..82d6b2e0 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnector.kt @@ -1,6 +1,9 @@ package ch.kleis.lcaac.core.datasource.csv +import ch.kleis.lcaac.core.config.ConnectorConfig import ch.kleis.lcaac.core.config.DataSourceConfig +import ch.kleis.lcaac.core.datasource.ConnectorBuilder +import ch.kleis.lcaac.core.datasource.ConnectorFactory import ch.kleis.lcaac.core.datasource.DataSourceConnector import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException import ch.kleis.lcaac.core.lang.evaluator.ToValue @@ -53,7 +56,8 @@ class CsvConnector( } override fun getAll(config: DataSourceConfig, source: DataSourceValue): Sequence> { - val location = config.location ?: throw IllegalArgumentException("Missing location in configuration for datasource '${config.name}'") + val location = config.location + ?: throw IllegalArgumentException("Missing location in configuration for datasource '${config.name}'") val records = load(location, source.schema) val filter = source.filter return records diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorBuilder.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorBuilder.kt new file mode 100644 index 00000000..b53a8674 --- /dev/null +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorBuilder.kt @@ -0,0 +1,13 @@ +package ch.kleis.lcaac.core.datasource.csv + +import ch.kleis.lcaac.core.config.ConnectorConfig +import ch.kleis.lcaac.core.datasource.ConnectorBuilder +import ch.kleis.lcaac.core.datasource.ConnectorFactory +import ch.kleis.lcaac.core.datasource.DataSourceConnector + +class CsvConnectorBuilder : ConnectorBuilder { + override fun buildOrNull(factory: ConnectorFactory, config: ConnectorConfig): DataSourceConnector? { + return config + .csv()?.let { CsvConnector(it, factory.getQuantityOperations()) } + } +} diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt index 180d268d..171239ee 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt @@ -56,9 +56,9 @@ class DefaultDataSourceOperationsTest { "mass" to QuantityFixture.twoKilograms, )), ) - val factory = object : ConnectorFactory { - override fun buildOrNull(config: ConnectorConfig): DataSourceConnector = connector - } + val builder = mockk>() + every { builder.buildOrNull(any(), any()) } returns connector + val factory = ConnectorFactory(config, ops, listOf(builder)) val sourceOps = DefaultDataSourceOperations(config, ops, factory) val source = DataSourceValue( config = DataSourceConfig( @@ -110,9 +110,9 @@ class DefaultDataSourceOperationsTest { "mass" to QuantityFixture.oneKilogram, )) - val factory = object : ConnectorFactory { - override fun buildOrNull(config: ConnectorConfig): DataSourceConnector = connector - } + val builder = mockk>() + every { builder.buildOrNull(any(), any()) } returns connector + val factory = ConnectorFactory(config, ops, listOf(builder)) val sourceOps = DefaultDataSourceOperations(config, ops, factory) val source = DataSourceValue( config = DataSourceConfig( @@ -164,9 +164,9 @@ class DefaultDataSourceOperationsTest { )), ) - val factory = object : ConnectorFactory { - override fun buildOrNull(config: ConnectorConfig): DataSourceConnector = connector - } + val builder = mockk>() + every { builder.buildOrNull(any(), any()) } returns connector + val factory = ConnectorFactory(config, ops, listOf(builder)) val sourceOps = DefaultDataSourceOperations(config, ops, factory) val source = DataSourceValue( config = DataSourceConfig( From 88c227ded58ba6e70d525419d1f9db29ef396528 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 3 Jun 2024 18:30:32 +0200 Subject: [PATCH 14/26] core: add csv connector by default in config --- .../main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt | 3 ++- .../kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt index 838a331e..ee8308f9 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/config/LcaacConfig.kt @@ -1,5 +1,6 @@ package ch.kleis.lcaac.core.config +import ch.kleis.lcaac.core.datasource.csv.CsvConnectorConfig import kotlinx.serialization.Serializable @Serializable @@ -7,7 +8,7 @@ data class LcaacConfig( val name: String = "", val description: String = "", val datasources: List = emptyList(), - val connectors: List = emptyList(), + val connectors: List = listOf(CsvConnectorConfig.default()), ) { private val datasourcesMap = datasources.associateBy { it.name } private val connectorsMap = connectors.associateBy { it.name } diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt index a1815889..deb0107b 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt @@ -12,6 +12,14 @@ data class CsvConnectorConfig( companion object { const val CSV_CONNECTOR_NAME = "csv" const val CSV_CONNECTOR_KEY_DIRECTORY = "directory" + + fun default(): ConnectorConfig = + ConnectorConfig( + name = CSV_CONNECTOR_NAME, + options = mapOf( + CSV_CONNECTOR_KEY_DIRECTORY to "." + ) + ) } } From 4f3fb0b1b15bf67df9f58bbc458f73d66a11ac18 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 3 Jun 2024 18:30:47 +0200 Subject: [PATCH 15/26] cli: default lcaac config when missing lcaac.yaml --- .../ch/kleis/lcaac/cli/cmd/AssessCommand.kt | 26 ++++++++++--------- .../ch/kleis/lcaac/cli/cmd/TestCommand.kt | 3 ++- .../ch/kleis/lcaac/cli/cmd/TraceCommand.kt | 6 +++-- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt index 476eb0af..660ad094 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt @@ -25,32 +25,34 @@ import java.nio.file.Path class AssessCommand : CliktCommand(name = "assess", help = "Returns the unitary impacts of a process in CSV format") { val name: String by argument().help("Process name") val labels: Map by option("-l", "--label") - .help( - """ + .help( + """ Specify a process label as a key value pair. Example: lcaac assess -l model="ABC" -l geo="FR". """.trimIndent()) - .associate() + .associate() private val getConfigPath = option("-c", "--config").file().default(File("lcaac.yaml")).help("Configuration file.") val configFile: File by getConfigPath val file: File? by option("-f", "--file").file(canBeDir = false) - .help(""" + .help(""" CSV file with parameter values. Example: `lcaac assess -f params.csv`. """.trimIndent()) val arguments: Map by option("-D", "--parameter") - .help( + .help( """ Override parameter value as a key value pair. Example: `lcaac assess -D x="12 kg" -D geo="UK" -f params.csv`. """.trimIndent()) - .associate() + .associate() override fun run() { - val config = configFile.inputStream().use { + val config = if (configFile.exists()) configFile.inputStream().use { Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) } + else LcaacConfig() + val files = lcaFiles(Path.of(".").toFile()) val symbolTable = Loader(BasicOperations).load(files, listOf(LoaderOption.WITH_PRELUDE)) val processor = CsvProcessor(config, symbolTable) @@ -72,7 +74,7 @@ class AssessCommand : CliktCommand(name = "assess", help = "Returns the unitary private fun loadRequests(): Iterator { return file?.let { loadRequestsFrom(it) } - ?: listOf(defaultRequest()).iterator() + ?: listOf(defaultRequest()).iterator() } private fun loadRequestsFrom(file: File): Iterator { @@ -85,10 +87,10 @@ class AssessCommand : CliktCommand(name = "assess", help = "Returns the unitary val header = pairs.mapIndexed { index, pair -> pair.first to index }.toMap() val record = pairs.map { it.second } return CsvRequest( - name, - labels, - header, - record, + name, + labels, + header, + record, ) } } diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt index 8d86c3b6..06cbf3e1 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt @@ -37,9 +37,10 @@ class TestCommand : CliktCommand(name = "test", help = "Run specified tests") { val showSuccess: Boolean by option("--show-success").flag(default = false).help("Show successful assertions") override fun run() { - val config = configFile.inputStream().use { + val config = if (configFile.exists()) configFile.inputStream().use { Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) } + else LcaacConfig() val ops = BasicOperations val sourceOps = DefaultDataSourceOperations(config, ops) diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt index a558d7bc..fc60a63c 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt @@ -51,9 +51,10 @@ class TraceCommand : CliktCommand(name = "trace", help = "Trace the contribution .associate() override fun run() { - val config = configFile.inputStream().use { + val config = if (configFile.exists()) configFile.inputStream().use { Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) } + else LcaacConfig() val ops = BasicOperations val sourceOps = DefaultDataSourceOperations(config, ops) @@ -97,7 +98,8 @@ class TraceCommand : CliktCommand(name = "trace", help = "Trace the contribution val demandedAmount = demandedProduct.quantity.amount val demandedUnit = demandedProduct.quantity.unit val demandedProductName = demandedProduct.product.name - val allocationAmount = (demandedProduct.allocation?.amount?.toDouble() ?: 1.0) * (demandedProduct.allocation?.unit?.scale ?: 1.0) + val allocationAmount = (demandedProduct.allocation?.amount?.toDouble() + ?: 1.0) * (demandedProduct.allocation?.unit?.scale ?: 1.0) observablePorts.asSequence() .map { row -> val supply = analysis.supplyOf(row) From c2345b12acf35d7188ccf139d8b53b1f024641db Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 3 Jun 2024 18:35:35 +0200 Subject: [PATCH 16/26] grammar: remove field primary_key --- grammar/src/main/antlr/LcaLang.g4 | 5 +---- grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt | 2 -- .../src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt | 3 +-- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/grammar/src/main/antlr/LcaLang.g4 b/grammar/src/main/antlr/LcaLang.g4 index a78d4b80..e292df9d 100644 --- a/grammar/src/main/antlr/LcaLang.g4 +++ b/grammar/src/main/antlr/LcaLang.g4 @@ -35,16 +35,13 @@ globalAssignment dataSourceDefinition : DATASOURCE_KEYWORD dataSourceRef LBRACE ( - locationField | primaryKeyField | schema | block_meta + locationField | schema | block_meta )* RBRACE ; locationField : LOCATION EQUAL STRING_LITERAL ; -primaryKeyField - : PRIMARY_KEY EQUAL STRING_LITERAL - ; schema : SCHEMA_KEYWORD LBRACE columnDefinition* diff --git a/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt b/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt index cf955d05..bb00ddcd 100644 --- a/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt +++ b/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreMapper.kt @@ -369,7 +369,6 @@ class CoreMapper( fun dataSourceDefinition(ctx: LcaLangParser.DataSourceDefinitionContext): EDataSource { val name = ctx.dataSourceRef().uid().ID().innerText() val location = ctx.locationField().firstOrNull()?.STRING_LITERAL()?.innerText() - val primaryKey = ctx.primaryKeyField().firstOrNull()?.STRING_LITERAL()?.innerText() val schemaBlock = ctx.schema().firstOrNull() ?: throw LoaderException("missing schema in datasource $name") val schema = schemaBlock.columnDefinition().associate { column -> val key = column.columnRef().innerText() @@ -387,7 +386,6 @@ class CoreMapper( DataSourceConfig( name = name, location = location, - primaryKey = primaryKey, options = options, ) ), diff --git a/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt b/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt index 5fe89cbb..ac7cf3e1 100644 --- a/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt +++ b/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreMapperTest.kt @@ -196,7 +196,6 @@ class CoreMapperTest { val ctx = LcaLangFixture.parser(""" datasource source { location = "file.csv" - primary_key = "geo" schema { mass = 1 kg geo = "FR" @@ -216,7 +215,7 @@ class CoreMapperTest { config = DataSourceConfig( name = "source", location = "file.csv", - primaryKey = "geo", + primaryKey = "id", options = mapOf( "description" to "This is a description", ) From 5b2a04ff9d710424ab8e9066d5df5b5ada867263 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 3 Jun 2024 19:52:40 +0200 Subject: [PATCH 17/26] grammar: removed unused keyword --- grammar/src/main/antlr/LcaLang.g4 | 1 - 1 file changed, 1 deletion(-) diff --git a/grammar/src/main/antlr/LcaLang.g4 b/grammar/src/main/antlr/LcaLang.g4 index e292df9d..9cebfa71 100644 --- a/grammar/src/main/antlr/LcaLang.g4 +++ b/grammar/src/main/antlr/LcaLang.g4 @@ -364,7 +364,6 @@ LABELS_KEYWORD : 'labels' ; DATASOURCE_KEYWORD : 'datasource' ; LOCATION : 'location' ; -PRIMARY_KEY : 'primary_key' ; SCHEMA_KEYWORD : 'schema' ; TEST_KEYWORD : 'test' ; From dcd888662ad0555cb8b4b74e3fcd0638230ff13e Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 3 Jun 2024 19:52:57 +0200 Subject: [PATCH 18/26] tutorials: removed lcaac.yaml file and undo modifications --- .../02-circular-footprint-formula/00-datasources.lca | 10 ++++++++++ .../02-circular-footprint-formula/lcaac.yaml | 6 ------ 2 files changed, 10 insertions(+), 6 deletions(-) delete mode 100644 tutorials/03-advanced/02-circular-footprint-formula/lcaac.yaml diff --git a/tutorials/03-advanced/02-circular-footprint-formula/00-datasources.lca b/tutorials/03-advanced/02-circular-footprint-formula/00-datasources.lca index 4a2dd549..fa8cdaef 100644 --- a/tutorials/03-advanced/02-circular-footprint-formula/00-datasources.lca +++ b/tutorials/03-advanced/02-circular-footprint-formula/00-datasources.lca @@ -3,6 +3,7 @@ */ datasource material_params { + location = "data/material_params.csv" schema { id = "material-01" A = 1.0 u @@ -31,6 +32,7 @@ datasource material_params { */ datasource R2_data { + location = "data/R2_data.csv" schema { id = "glass-01" geo = "GLO" @@ -45,6 +47,7 @@ datasource R2_data { // VIRGIN datasource Ev_data { + location = "data/Ev_data.csv" schema { id = "material-01" GWP = 1 kg_CO2_Eq @@ -68,6 +71,7 @@ process virgin_production_process { // RECYCLED datasource Erecycled_data { + location = "data/Erecycled_data.csv" schema { id = "material-01" GWP = 1 kg_CO2_Eq @@ -91,6 +95,7 @@ process upstream_recycling_process { // RECYCLING END OF LIFE datasource ErecyclingEol_data { + location = "data/ErecyclingEol_data.csv" schema { id = "material-01" GWP = 1 kg_CO2_Eq @@ -114,6 +119,7 @@ process downstream_recycling_process { // SUBSTITUTE FOR VIRGIN datasource EstarV_data { + location = "data/EstarV_data.csv" schema { id = "material-01" GWP = 1 kg_CO2_Eq @@ -137,6 +143,7 @@ process substitute_production_process { // ENERGY RECOVERY datasource Eer_data { + location = "data/Eer_data.csv" schema { id = "material-01" GWP = 1 kg_CO2_Eq @@ -160,6 +167,7 @@ process energy_recovery_process { // HEAT PRODUCTION datasource Ese_heat_data { + location = "data/Ese_heat_data.csv" schema { geo = "GLO" GWP = 1 kg_CO2_Eq @@ -183,6 +191,7 @@ process heat_production_process { // ELECTRICITY PRODUCTION datasource Ese_elec_data { + location = "data/Ese_elec_data.csv" schema { geo = "GLO" GWP = 1 kg_CO2_Eq @@ -206,6 +215,7 @@ process elec_production_process { // DISPOSAL datasource Ed_data { + location = "data/Ed_data.csv" schema { id = "material-01" GWP = 1 kg_CO2_Eq diff --git a/tutorials/03-advanced/02-circular-footprint-formula/lcaac.yaml b/tutorials/03-advanced/02-circular-footprint-formula/lcaac.yaml deleted file mode 100644 index 571a0ba0..00000000 --- a/tutorials/03-advanced/02-circular-footprint-formula/lcaac.yaml +++ /dev/null @@ -1,6 +0,0 @@ -name: Circular footprint formula -description: A sample project to illustrate the implementation of the circular footprint formula. -connectors: - - name: csv - options: - directory: "data" From bb181f69a7f694af8af92ccc23de21683da11a5a Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 3 Jun 2024 23:07:46 +0200 Subject: [PATCH 19/26] core: added working directory to connector factory --- .../ch/kleis/lcaac/core/datasource/ConnectorFactory.kt | 2 ++ .../lcaac/core/datasource/DefaultDataSourceOperations.kt | 3 ++- .../kleis/lcaac/core/datasource/csv/CsvConnectorBuilder.kt | 2 +- .../kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt | 7 ++++--- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorFactory.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorFactory.kt index aa017658..c88fe50a 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorFactory.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/ConnectorFactory.kt @@ -6,12 +6,14 @@ import ch.kleis.lcaac.core.datasource.csv.CsvConnectorBuilder import ch.kleis.lcaac.core.math.QuantityOperations class ConnectorFactory( + private val workingDirectory: String, private val lcaacConfig: LcaacConfig, private val ops: QuantityOperations, builders: List> = listOf(CsvConnectorBuilder()) ) { private val builders = ArrayList>(builders) + fun getWorkingDirectory(): String = workingDirectory fun getLcaacConfig(): LcaacConfig = lcaacConfig fun getQuantityOperations(): QuantityOperations = ops diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt index 53ed0fa3..8e4f1a06 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt @@ -15,7 +15,8 @@ import ch.kleis.lcaac.core.prelude.Prelude class DefaultDataSourceOperations( private val config: LcaacConfig, private val ops: QuantityOperations, - private val connectorFactory: ConnectorFactory = ConnectorFactory(config, ops) + private val workingDirectory: String, + private val connectorFactory: ConnectorFactory = ConnectorFactory(workingDirectory, config, ops) ) : DataSourceOperations { private val connectors = config.connectors .mapNotNull { connectorFactory.buildOrNull(it) } diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorBuilder.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorBuilder.kt index b53a8674..ef018fbb 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorBuilder.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorBuilder.kt @@ -8,6 +8,6 @@ import ch.kleis.lcaac.core.datasource.DataSourceConnector class CsvConnectorBuilder : ConnectorBuilder { override fun buildOrNull(factory: ConnectorFactory, config: ConnectorConfig): DataSourceConnector? { return config - .csv()?.let { CsvConnector(it, factory.getQuantityOperations()) } + .csv(factory.getWorkingDirectory())?.let { CsvConnector(it, factory.getQuantityOperations()) } } } diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt index deb0107b..670ffaf2 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt @@ -5,6 +5,7 @@ import ch.kleis.lcaac.core.datasource.csv.CsvConnectorConfig.Companion.CSV_CONNE import ch.kleis.lcaac.core.datasource.csv.CsvConnectorConfig.Companion.CSV_CONNECTOR_NAME import java.io.File import java.nio.file.Path +import java.nio.file.Paths data class CsvConnectorConfig( val directory: File, @@ -23,13 +24,13 @@ data class CsvConnectorConfig( } } -fun ConnectorConfig.csv(): CsvConnectorConfig? { +fun ConnectorConfig.csv(workingDirectory: String): CsvConnectorConfig? { if (this.name != CSV_CONNECTOR_NAME) { return null } val directory = this.options[CSV_CONNECTOR_KEY_DIRECTORY] - ?.let { Path.of(it).toFile() } - ?: Path.of(".").toFile() + ?.let { Paths.get(workingDirectory, it).toFile() } + ?: Path.of(workingDirectory).toFile() return CsvConnectorConfig( directory, ) From a04466186a076d46e752e6f947c30f7231fddd39 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 3 Jun 2024 23:08:19 +0200 Subject: [PATCH 20/26] cli: option -p to specify project folder or yaml file --- .../ch/kleis/lcaac/cli/cmd/AssessCommand.kt | 19 +++++++++++------- .../ch/kleis/lcaac/cli/cmd/TestCommand.kt | 19 ++++++++++++------ .../ch/kleis/lcaac/cli/cmd/TraceCommand.kt | 20 ++++++++++++------- .../kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt | 2 ++ .../ch/kleis/lcaac/cli/csv/CsvProcessor.kt | 3 ++- 5 files changed, 42 insertions(+), 21 deletions(-) diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt index 660ad094..e558e97b 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt @@ -19,9 +19,9 @@ import com.github.ajalt.clikt.parameters.options.help import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.file import java.io.File -import java.nio.file.Path +import java.nio.file.Paths -@Suppress("MemberVisibilityCanBePrivate") +@Suppress("MemberVisibilityCanBePrivate", "DuplicatedCode") class AssessCommand : CliktCommand(name = "assess", help = "Returns the unitary impacts of a process in CSV format") { val name: String by argument().help("Process name") val labels: Map by option("-l", "--label") @@ -31,8 +31,10 @@ class AssessCommand : CliktCommand(name = "assess", help = "Returns the unitary Example: lcaac assess -l model="ABC" -l geo="FR". """.trimIndent()) .associate() - private val getConfigPath = option("-c", "--config").file().default(File("lcaac.yaml")).help("Configuration file.") - val configFile: File by getConfigPath + private val getProjectPath = option("-p", "--project").file() + .default(File(defaultLcaacFilename)) + .help("Path to project folder or yaml file.") + val projectPath: File by getProjectPath val file: File? by option("-f", "--file").file(canBeDir = false) .help(""" @@ -48,14 +50,17 @@ class AssessCommand : CliktCommand(name = "assess", help = "Returns the unitary .associate() override fun run() { - val config = if (configFile.exists()) configFile.inputStream().use { + val workingDirectory = if (projectPath.isDirectory) projectPath else projectPath.parentFile + val lcaacConfigFile = if (projectPath.isDirectory) Paths.get(workingDirectory.path, defaultLcaacFilename).toFile() + else projectPath + val config = if (lcaacConfigFile.exists()) projectPath.inputStream().use { Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) } else LcaacConfig() - val files = lcaFiles(Path.of(".").toFile()) + val files = lcaFiles(workingDirectory) val symbolTable = Loader(BasicOperations).load(files, listOf(LoaderOption.WITH_PRELUDE)) - val processor = CsvProcessor(config, symbolTable) + val processor = CsvProcessor(config, symbolTable, workingDirectory.path) val iterator = loadRequests() val writer = CsvResultWriter() var first = true diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt index 06cbf3e1..effa4bb1 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt @@ -21,14 +21,18 @@ import com.github.ajalt.clikt.parameters.options.help import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.file import java.io.File -import java.nio.file.Path +import java.nio.file.Paths private const val greenTick = "\u2705" private const val redCross = "\u274C" +@Suppress("MemberVisibilityCanBePrivate", "DuplicatedCode") class TestCommand : CliktCommand(name = "test", help = "Run specified tests") { - private val getConfigPath = option("-c", "--config").file().default(File("lcaac.yaml")).help("Configuration file.") - val configFile: File by getConfigPath + private val getProjectPath = option("-p", "--project").file() + .default(File(defaultLcaacFilename)) + .help("Path to project folder or yaml file.") + val projectPath: File by getProjectPath + val file: File? by option("-f", "--file").file(canBeDir = false) .help(""" CSV file with parameter values. @@ -37,15 +41,18 @@ class TestCommand : CliktCommand(name = "test", help = "Run specified tests") { val showSuccess: Boolean by option("--show-success").flag(default = false).help("Show successful assertions") override fun run() { - val config = if (configFile.exists()) configFile.inputStream().use { + val workingDirectory = if (projectPath.isDirectory) projectPath else projectPath.parentFile + val lcaacConfigFile = if (projectPath.isDirectory) Paths.get(workingDirectory.path, defaultLcaacFilename).toFile() + else projectPath + val config = if (lcaacConfigFile.exists()) projectPath.inputStream().use { Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) } else LcaacConfig() val ops = BasicOperations - val sourceOps = DefaultDataSourceOperations(config, ops) + val sourceOps = DefaultDataSourceOperations(config, ops, workingDirectory.path) - val files = lcaFiles(Path.of(".").toFile()) + val files = lcaFiles(workingDirectory) val symbolTable = Loader(ops).load(files, listOf(LoaderOption.WITH_PRELUDE)) val mapper = CoreTestMapper() val cases = files diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt index fc60a63c..b4eac258 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt @@ -28,9 +28,9 @@ import com.github.ajalt.clikt.parameters.types.file import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVPrinter import java.io.File -import java.nio.file.Path +import java.nio.file.Paths -@Suppress("MemberVisibilityCanBePrivate") +@Suppress("MemberVisibilityCanBePrivate", "DuplicatedCode") class TraceCommand : CliktCommand(name = "trace", help = "Trace the contributions") { val name: String by argument().help("Process name") val labels: Map by option("-l", "--label") @@ -40,8 +40,11 @@ class TraceCommand : CliktCommand(name = "trace", help = "Trace the contribution Example: lcaac assess -l model="ABC" -l geo="FR". """.trimIndent()) .associate() - private val getConfigPath = option("-c", "--config").file().default(File("lcaac.yaml")).help("Configuration file.") - val configFile: File by getConfigPath + private val getProjectPath = option("-p", "--project").file() + .default(File(defaultLcaacFilename)) + .help("Path to project folder or yaml file.") + val projectPath: File by getProjectPath + val arguments: Map by option("-D", "--parameter") .help( """ @@ -51,14 +54,17 @@ class TraceCommand : CliktCommand(name = "trace", help = "Trace the contribution .associate() override fun run() { - val config = if (configFile.exists()) configFile.inputStream().use { + val workingDirectory = if (projectPath.isDirectory) projectPath else projectPath.parentFile + val lcaacConfigFile = if (projectPath.isDirectory) Paths.get(workingDirectory.path, defaultLcaacFilename).toFile() + else projectPath + val config = if (lcaacConfigFile.exists()) projectPath.inputStream().use { Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) } else LcaacConfig() val ops = BasicOperations - val sourceOps = DefaultDataSourceOperations(config, ops) + val sourceOps = DefaultDataSourceOperations(config, ops, workingDirectory.path) - val files = lcaFiles(Path.of(".").toFile()) + val files = lcaFiles(workingDirectory) val symbolTable = Loader(ops).load(files, listOf(LoaderOption.WITH_PRELUDE)) val evaluator = Evaluator(symbolTable, ops, sourceOps) val template = symbolTable.getTemplate(name, labels) diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt index 90d037bc..d8c0f70d 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt @@ -18,6 +18,8 @@ import java.lang.Double.parseDouble import java.nio.file.Files import kotlin.io.path.isRegularFile +val defaultLcaacFilename = "lcaac.yaml" + fun lcaFiles(root: File): Sequence { return Files.walk(root.toPath()) .filter { it.isRegularFile() } diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessor.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessor.kt index c56942be..77fa54b8 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessor.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessor.kt @@ -14,9 +14,10 @@ import ch.kleis.lcaac.core.math.basic.BasicOperations class CsvProcessor( config: LcaacConfig, private val symbolTable: SymbolTable, + workingDirectory: String, ) { private val ops = BasicOperations - private val sourceOps = DefaultDataSourceOperations(config, ops) + private val sourceOps = DefaultDataSourceOperations(config, ops, workingDirectory) private val dataReducer = DataExpressionReducer(symbolTable.data, symbolTable.dataSources, ops, sourceOps) private val evaluator = Evaluator(symbolTable, ops, sourceOps) From d16947818b4fe2310840d5ba2b11238024928430 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 3 Jun 2024 23:52:52 +0200 Subject: [PATCH 21/26] core: fix test --- .../datasource/DefaultDataSourceOperationsTest.kt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt index 171239ee..c4cdd6b4 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt @@ -1,8 +1,8 @@ package ch.kleis.lcaac.core.datasource -import ch.kleis.lcaac.core.config.LcaacConfig import ch.kleis.lcaac.core.config.ConnectorConfig import ch.kleis.lcaac.core.config.DataSourceConfig +import ch.kleis.lcaac.core.config.LcaacConfig import ch.kleis.lcaac.core.lang.expression.EQuantityScale import ch.kleis.lcaac.core.lang.expression.ERecord import ch.kleis.lcaac.core.lang.expression.EStringLiteral @@ -58,8 +58,8 @@ class DefaultDataSourceOperationsTest { ) val builder = mockk>() every { builder.buildOrNull(any(), any()) } returns connector - val factory = ConnectorFactory(config, ops, listOf(builder)) - val sourceOps = DefaultDataSourceOperations(config, ops, factory) + val factory = ConnectorFactory(".", config, ops, listOf(builder)) + val sourceOps = DefaultDataSourceOperations(config, ops, ".", factory) val source = DataSourceValue( config = DataSourceConfig( name = sourceName, @@ -112,8 +112,8 @@ class DefaultDataSourceOperationsTest { val builder = mockk>() every { builder.buildOrNull(any(), any()) } returns connector - val factory = ConnectorFactory(config, ops, listOf(builder)) - val sourceOps = DefaultDataSourceOperations(config, ops, factory) + val factory = ConnectorFactory(".", config, ops, listOf(builder)) + val sourceOps = DefaultDataSourceOperations(config, ops, ".", factory) val source = DataSourceValue( config = DataSourceConfig( name = sourceName, @@ -166,8 +166,8 @@ class DefaultDataSourceOperationsTest { val builder = mockk>() every { builder.buildOrNull(any(), any()) } returns connector - val factory = ConnectorFactory(config, ops, listOf(builder)) - val sourceOps = DefaultDataSourceOperations(config, ops, factory) + val factory = ConnectorFactory(".", config, ops, listOf(builder)) + val sourceOps = DefaultDataSourceOperations(config, ops, ".", factory) val source = DataSourceValue( config = DataSourceConfig( name = sourceName, From 8dee67bedcdb3966ba8ca92964cbd42959cb4a63 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 3 Jun 2024 23:53:12 +0200 Subject: [PATCH 22/26] cli: smarter parsing of project path --- .../ch/kleis/lcaac/cli/cmd/AssessCommand.kt | 4 +- .../ch/kleis/lcaac/cli/cmd/TestCommand.kt | 12 +++-- .../ch/kleis/lcaac/cli/cmd/TraceCommand.kt | 5 +- .../kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt | 12 ++++- .../ch/kleis/lcaac/cli/cmd/UtilsKtTest.kt | 53 +++++++++++++++++++ .../kleis/lcaac/cli/csv/CsvProcessorTest.kt | 6 +-- 6 files changed, 77 insertions(+), 15 deletions(-) diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt index e558e97b..52eebc2b 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/AssessCommand.kt @@ -50,9 +50,7 @@ class AssessCommand : CliktCommand(name = "assess", help = "Returns the unitary .associate() override fun run() { - val workingDirectory = if (projectPath.isDirectory) projectPath else projectPath.parentFile - val lcaacConfigFile = if (projectPath.isDirectory) Paths.get(workingDirectory.path, defaultLcaacFilename).toFile() - else projectPath + val (workingDirectory, lcaacConfigFile) = parseProjectPath(projectPath) val config = if (lcaacConfigFile.exists()) projectPath.inputStream().use { Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) } diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt index effa4bb1..1ea63725 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TestCommand.kt @@ -15,19 +15,23 @@ import com.charleskorn.kaml.Yaml import com.charleskorn.kaml.decodeFromStream import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.ProgramResult +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.arguments.default +import com.github.ajalt.clikt.parameters.arguments.help import com.github.ajalt.clikt.parameters.options.default import com.github.ajalt.clikt.parameters.options.flag import com.github.ajalt.clikt.parameters.options.help import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.file import java.io.File -import java.nio.file.Paths private const val greenTick = "\u2705" private const val redCross = "\u274C" @Suppress("MemberVisibilityCanBePrivate", "DuplicatedCode") class TestCommand : CliktCommand(name = "test", help = "Run specified tests") { + val name: String by argument().help("Process name").default("") + private val getProjectPath = option("-p", "--project").file() .default(File(defaultLcaacFilename)) .help("Path to project folder or yaml file.") @@ -41,9 +45,8 @@ class TestCommand : CliktCommand(name = "test", help = "Run specified tests") { val showSuccess: Boolean by option("--show-success").flag(default = false).help("Show successful assertions") override fun run() { - val workingDirectory = if (projectPath.isDirectory) projectPath else projectPath.parentFile - val lcaacConfigFile = if (projectPath.isDirectory) Paths.get(workingDirectory.path, defaultLcaacFilename).toFile() - else projectPath + val (workingDirectory, lcaacConfigFile) = parseProjectPath(projectPath) + val config = if (lcaacConfigFile.exists()) projectPath.inputStream().use { Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) } @@ -58,6 +61,7 @@ class TestCommand : CliktCommand(name = "test", help = "Run specified tests") { val cases = files .flatMap { it.testDefinition() } .map { mapper.test(it) } + .filter { name.isBlank() || it.name == name } val runner = BasicTestRunner( symbolTable, sourceOps, diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt index b4eac258..96eb14e5 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt @@ -28,7 +28,6 @@ import com.github.ajalt.clikt.parameters.types.file import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVPrinter import java.io.File -import java.nio.file.Paths @Suppress("MemberVisibilityCanBePrivate", "DuplicatedCode") class TraceCommand : CliktCommand(name = "trace", help = "Trace the contributions") { @@ -54,9 +53,7 @@ class TraceCommand : CliktCommand(name = "trace", help = "Trace the contribution .associate() override fun run() { - val workingDirectory = if (projectPath.isDirectory) projectPath else projectPath.parentFile - val lcaacConfigFile = if (projectPath.isDirectory) Paths.get(workingDirectory.path, defaultLcaacFilename).toFile() - else projectPath + val (workingDirectory, lcaacConfigFile) = parseProjectPath(projectPath) val config = if (lcaacConfigFile.exists()) projectPath.inputStream().use { Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) } diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt index d8c0f70d..ea0bf7b9 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt @@ -16,9 +16,19 @@ import java.io.File import java.io.InputStream import java.lang.Double.parseDouble import java.nio.file.Files +import kotlin.io.path.Path import kotlin.io.path.isRegularFile -val defaultLcaacFilename = "lcaac.yaml" +const val defaultLcaacFilename = "lcaac.yaml" + +fun parseProjectPath(path: File): Pair { + if (path.isDirectory) { + val configFile = Path(defaultLcaacFilename).toFile() + return path to configFile + } + val workingDirectory = path.parentFile ?: Path(".").toFile() + return workingDirectory to path +} fun lcaFiles(root: File): Sequence { return Files.walk(root.toPath()) diff --git a/cli/src/test/kotlin/ch/kleis/lcaac/cli/cmd/UtilsKtTest.kt b/cli/src/test/kotlin/ch/kleis/lcaac/cli/cmd/UtilsKtTest.kt index b8b89af0..c455f8d6 100644 --- a/cli/src/test/kotlin/ch/kleis/lcaac/cli/cmd/UtilsKtTest.kt +++ b/cli/src/test/kotlin/ch/kleis/lcaac/cli/cmd/UtilsKtTest.kt @@ -6,13 +6,66 @@ import ch.kleis.lcaac.core.lang.expression.EQuantityMul import ch.kleis.lcaac.core.lang.expression.EQuantityScale import ch.kleis.lcaac.core.math.basic.BasicNumber import ch.kleis.lcaac.core.prelude.Prelude +import io.mockk.every +import io.mockk.mockk import org.junit.jupiter.api.assertThrows +import java.io.File +import java.nio.file.Path +import java.nio.file.Paths +import kotlin.io.path.Path import kotlin.test.Test import kotlin.test.assertEquals class UtilsKtTest { + @Test + fun parseProjectPath_whenSimpleFile() { + // given + val path = mockk() + every { path.isDirectory } returns false + every { path.parentFile } returns null + every { path.path } returns "lcaac.yaml" + + // when + val (workingDir, configFile) = parseProjectPath(path) + + // then + assertEquals(".", workingDir.path) + assertEquals("lcaac.yaml", configFile.path) + } + + @Test + fun parseProjectPath_whenFileWithParentDirectory() { + // given + val path = mockk() + every { path.isDirectory } returns false + every { path.parentFile } returns Paths.get("some", "directory").toFile() + every { path.path } returns "lcaac.yaml" + + // when + val (workingDir, configFile) = parseProjectPath(path) + + // then + assertEquals("some/directory", workingDir.path) + assertEquals("lcaac.yaml", configFile.path) + } + + @Test + fun parseProjectPath_whenDirectory() { + // given + val path = mockk() + every { path.isDirectory } returns true + every { path.path } returns "some/directory" + + // when + val (workingDir, configFile) = parseProjectPath(path) + + // then + assertEquals("some/directory", workingDir.path) + assertEquals("lcaac.yaml", configFile.path) + } + @Test fun parseQuantityWithDefaultUnit_invalidExpression() { // given diff --git a/cli/src/test/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessorTest.kt b/cli/src/test/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessorTest.kt index 166c23e1..daab872a 100644 --- a/cli/src/test/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessorTest.kt +++ b/cli/src/test/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessorTest.kt @@ -42,7 +42,7 @@ class CsvProcessorTest { """.trimIndent() val symbolTable = load(content) val config = LcaacConfig() - val processor = CsvProcessor(config, symbolTable) + val processor = CsvProcessor(config, symbolTable, ".") val request = CsvRequest( "main", emptyMap(), @@ -106,7 +106,7 @@ class CsvProcessorTest { """.trimIndent() val symbolTable = load(content) val config = LcaacConfig() - val processor = CsvProcessor(config, symbolTable) + val processor = CsvProcessor(config, symbolTable, ".") val request = CsvRequest( "main", emptyMap(), @@ -162,7 +162,7 @@ class CsvProcessorTest { """.trimIndent() val symbolTable = load(content) val config = LcaacConfig() - val processor = CsvProcessor(config, symbolTable) + val processor = CsvProcessor(config, symbolTable, ".") val request = CsvRequest( "main", emptyMap(), From 3ce9a5eb6d35e1c78ebc8bd44f7ca289dc704106 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 3 Jun 2024 23:53:33 +0200 Subject: [PATCH 23/26] tutorials: added tutorial for project configuration file --- .../03-advanced/03-project-file/README.md | 34 ++++++++++++++++++ .../03-project-file/data/inventory.csv | 5 +++ .../03-project-file/lcaac-mock.yaml | 6 ++++ .../03-advanced/03-project-file/lcaac.yaml | 6 ++++ .../03-advanced/03-project-file/main.lca | 36 +++++++++++++++++++ .../03-project-file/mock/inventory.csv | 5 +++ 6 files changed, 92 insertions(+) create mode 100644 tutorials/03-advanced/03-project-file/README.md create mode 100644 tutorials/03-advanced/03-project-file/data/inventory.csv create mode 100644 tutorials/03-advanced/03-project-file/lcaac-mock.yaml create mode 100644 tutorials/03-advanced/03-project-file/lcaac.yaml create mode 100644 tutorials/03-advanced/03-project-file/main.lca create mode 100644 tutorials/03-advanced/03-project-file/mock/inventory.csv diff --git a/tutorials/03-advanced/03-project-file/README.md b/tutorials/03-advanced/03-project-file/README.md new file mode 100644 index 00000000..6d3c404e --- /dev/null +++ b/tutorials/03-advanced/03-project-file/README.md @@ -0,0 +1,34 @@ +# Project description file + +A LCA as Code project can be specified with a YAML configuration. +In this tutorial, the configuration file is used to specify: +- the project name +- the project description +- and customizing the CSV connector + +Hence, the file `lcaac.yaml` contains +```yaml +name: Project file +description: Sample project to illustrate the use of lcaac.yaml +connectors: + - name: csv + options: + directory: data +``` + +while the file `lcaac-mock.yaml` contains +```yaml +name: Project file +description: Sample project to illustrate the use of lcaac.yaml +connectors: + - name: csv + options: + directory: mock +``` + +Here each file specifies a different location for the folder containing the CSV files supporting the datasources. +You can choose which settings to use with the cli option `-p` or `--project`. +```bash +lcaac assess --project lcaac.yaml main +lcaac assess --project lcaac-mock.yaml main +``` diff --git a/tutorials/03-advanced/03-project-file/data/inventory.csv b/tutorials/03-advanced/03-project-file/data/inventory.csv new file mode 100644 index 00000000..f51b387c --- /dev/null +++ b/tutorials/03-advanced/03-project-file/data/inventory.csv @@ -0,0 +1,5 @@ +id,weight +s01,1 +s02,2 +s03,3 +s04,4 diff --git a/tutorials/03-advanced/03-project-file/lcaac-mock.yaml b/tutorials/03-advanced/03-project-file/lcaac-mock.yaml new file mode 100644 index 00000000..23a094a1 --- /dev/null +++ b/tutorials/03-advanced/03-project-file/lcaac-mock.yaml @@ -0,0 +1,6 @@ +name: Project file +description: Sample project to illustrate the use of lcaac.yaml +connectors: + - name: csv + options: + directory: mock diff --git a/tutorials/03-advanced/03-project-file/lcaac.yaml b/tutorials/03-advanced/03-project-file/lcaac.yaml new file mode 100644 index 00000000..029fd63f --- /dev/null +++ b/tutorials/03-advanced/03-project-file/lcaac.yaml @@ -0,0 +1,6 @@ +name: Project file +description: Sample project to illustrate the use of lcaac.yaml +connectors: + - name: csv + options: + directory: data diff --git a/tutorials/03-advanced/03-project-file/main.lca b/tutorials/03-advanced/03-project-file/main.lca new file mode 100644 index 00000000..91e269fa --- /dev/null +++ b/tutorials/03-advanced/03-project-file/main.lca @@ -0,0 +1,36 @@ +datasource inventory { + location = "inventory.csv" + schema { + id = "s01" + weight = 0 kg + } +} + +process main { + products { + 1 kg main + } + impacts { + for_each item from inventory { + item.weight mass + } + } +} + +test main_with_data { + given { + 1 kg main from main + } + assert { + mass between 10 kg and 10 kg + } +} + +test main_with_mock_data { + given { + 1 kg main from main + } + assert { + mass between 100 kg and 100 kg + } +} diff --git a/tutorials/03-advanced/03-project-file/mock/inventory.csv b/tutorials/03-advanced/03-project-file/mock/inventory.csv new file mode 100644 index 00000000..9a518d20 --- /dev/null +++ b/tutorials/03-advanced/03-project-file/mock/inventory.csv @@ -0,0 +1,5 @@ +id,weight +s01,10 +s02,20 +s03,30 +s04,40 From 3112357a2d6949d027c1da1ad5b4ac650920829f Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 3 Jun 2024 23:54:45 +0200 Subject: [PATCH 24/26] tutorials: more test cases in run.sh --- tutorials/run.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tutorials/run.sh b/tutorials/run.sh index a8c15bbd..2f5caf5a 100755 --- a/tutorials/run.sh +++ b/tutorials/run.sh @@ -33,6 +33,9 @@ lcaac test -p $TUTORIALS_PATH/02-language-features/05-datasources lcaac test -p $TUTORIALS_PATH/03-advanced/01-relational-modeling lcaac test -p $TUTORIALS_PATH/03-advanced/02-circular-footprint-formula +lcaac test -p $TUTORIALS_PATH/03-advanced/03-project-file/lcaac.yaml main_with_data +lcaac test -p $TUTORIALS_PATH/03-advanced/03-project-file/lcaac-mock.yaml main_with_mock_data + # Check custom dimensions tutorial set -euo # The following assessment is expected to fail. From e4798151a6ca5f147d290e9ebff7d1fb4591ff8a Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 10 Jun 2024 09:49:03 +0200 Subject: [PATCH 25/26] bump version 1.6.5 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 359abbdf..7af6cd39 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,4 +2,4 @@ javaVersion=17 gradleVersion=7.6 org.gradle.jvmargs=-Xmx4096m lcaacGroup=ch.kleis.lcaac -lcaacVersion=1.6.4 +lcaacVersion=1.6.5 From d2cfb451ef8289547a686cfc17be0a0e27e66a26 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 10 Jun 2024 16:28:38 +0200 Subject: [PATCH 26/26] core: csv connector config: smarter concat of paths --- .../core/datasource/csv/CsvConnectorConfig.kt | 7 +- .../csv/CsvConnectorConfigKtTest.kt | 112 ++++++++++++++++++ 2 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 core/src/test/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfigKtTest.kt diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt index 670ffaf2..ee7cba81 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfig.kt @@ -6,6 +6,7 @@ import ch.kleis.lcaac.core.datasource.csv.CsvConnectorConfig.Companion.CSV_CONNE import java.io.File import java.nio.file.Path import java.nio.file.Paths +import kotlin.io.path.pathString data class CsvConnectorConfig( val directory: File, @@ -29,7 +30,11 @@ fun ConnectorConfig.csv(workingDirectory: String): CsvConnectorConfig? { return null } val directory = this.options[CSV_CONNECTOR_KEY_DIRECTORY] - ?.let { Paths.get(workingDirectory, it).toFile() } + ?.let { + val directoryPath = Path.of(it) + if (directoryPath.isAbsolute) directoryPath.toFile() + else Paths.get(workingDirectory, it).toFile() + } ?: Path.of(workingDirectory).toFile() return CsvConnectorConfig( directory, diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfigKtTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfigKtTest.kt new file mode 100644 index 00000000..609e1ade --- /dev/null +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/csv/CsvConnectorConfigKtTest.kt @@ -0,0 +1,112 @@ +package ch.kleis.lcaac.core.datasource.csv + +import ch.kleis.lcaac.core.config.ConnectorConfig +import org.junit.jupiter.api.Assertions.* +import java.io.File +import kotlin.io.path.Path +import kotlin.test.Test + +class CsvConnectorConfigKtTest { + @Test + fun csv() { + // given + val workingDirectory = "/foo" + val config = ConnectorConfig( + name = "csv", + options = mapOf( + "directory" to "bar" + ) + ) + + // when + val actual = config.csv(workingDirectory) + + // then + val expected = CsvConnectorConfig( + directory = Path("/foo/bar").toFile(), + ) + assertEquals(expected, actual) + } + + @Test + fun csv_whenTwoAbsolutePaths() { + // given + val workingDirectory = "/foo" + val config = ConnectorConfig( + name = "csv", + options = mapOf( + "directory" to "/foo/bar" + ) + ) + + // when + val actual = config.csv(workingDirectory) + + // then + val expected = CsvConnectorConfig( + directory = Path("/foo/bar").toFile(), + ) + assertEquals(expected, actual) + } + + @Test + fun csv_whenRelativeSubfolder() { + // given + val workingDirectory = "/foo" + val config = ConnectorConfig( + name = "csv", + options = mapOf( + "directory" to "baz/bar" + ) + ) + + // when + val actual = config.csv(workingDirectory) + + // then + val expected = CsvConnectorConfig( + directory = Path("/foo/baz/bar").toFile(), + ) + assertEquals(expected, actual) + } + + @Test + fun csv_whenIncompatibleAbsolutePaths_thenTrustLocation() { + // given + val workingDirectory = "/foo/dot" + val config = ConnectorConfig( + name = "csv", + options = mapOf( + "directory" to "/baz/bar" + ) + ) + + // when + val actual = config.csv(workingDirectory) + + // then + val expected = CsvConnectorConfig( + directory = Path("/baz/bar").toFile(), + ) + assertEquals(expected, actual) + } + + @Test + fun csv_whenEmptyOptions() { + // given + val workingDirectory = "/foo/dot" + val config = ConnectorConfig( + name = "csv", + options = emptyMap(), + ) + + // when + val actual = config.csv(workingDirectory) + + // then + val expected = CsvConnectorConfig( + directory = Path("/foo/dot").toFile(), + ) + assertEquals(expected, actual) + } +}