diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt index 6e08de24..017a2504 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt @@ -139,7 +139,7 @@ class DefaultDataSourceOperations( }.reduce { acc, expression -> reducer.reduce(EQuantityMul(acc, expression)) } - }.fold(zero as DataExpression, ({ acc, expression -> reducer.reduce(EQuantityAdd(acc, expression)) })) - + }.fold(zero as DataExpression, ({ acc, expression -> + reducer.reduce(EQuantityAdd(acc, expression)) })) } } diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryConnector.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryConnector.kt index 12ae101d..e23f4ed8 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryConnector.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryConnector.kt @@ -5,9 +5,9 @@ import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.datasource.DataSourceConnector import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException import ch.kleis.lcaac.core.lang.expression.ERecord -import ch.kleis.lcaac.core.lang.expression.EStringLiteral import ch.kleis.lcaac.core.lang.value.DataSourceValue import ch.kleis.lcaac.core.lang.value.DataValue +import ch.kleis.lcaac.core.lang.value.RecordValue import ch.kleis.lcaac.core.lang.value.StringValue class InMemoryConnector( @@ -31,7 +31,9 @@ class InMemoryConnector( ?.records ?.filter(applyFilter(filter)) ?: emptyList() - return records.asSequence() + return records + .map { it.toERecord() } + .asSequence() } override fun getFirst(config: DataSourceConfig, source: DataSourceValue): ERecord { @@ -42,12 +44,12 @@ class InMemoryConnector( private fun applyFilter( filter: Map>, -): (ERecord) -> Boolean = { record -> +): (RecordValue) -> Boolean = { record -> filter.entries.all { val expected = it.value if (expected is StringValue) { when (val v = record.entries[it.key]) { - is EStringLiteral -> expected.s == v.value + is StringValue -> expected.s == v.s else -> throw EvaluatorException("invalid type for column '${it.key}': expected 'EStringLiteral', found '${v?.javaClass?.simpleName}'") } } else throw EvaluatorException("invalid matching condition $it") diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryDatasource.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryDatasource.kt index 05f50b55..25c54d48 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryDatasource.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryDatasource.kt @@ -1,7 +1,7 @@ package ch.kleis.lcaac.core.datasource.in_memory -import ch.kleis.lcaac.core.lang.expression.ERecord +import ch.kleis.lcaac.core.lang.value.RecordValue data class InMemoryDatasource( - val records: List>, + val records: List>, ) diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataValue.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataValue.kt index a390d26b..06ae79ca 100644 --- a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataValue.kt +++ b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataValue.kt @@ -1,6 +1,7 @@ package ch.kleis.lcaac.core.lang.value import ch.kleis.lcaac.core.lang.expression.EQuantityScale +import ch.kleis.lcaac.core.lang.expression.ERecord import ch.kleis.lcaac.core.lang.expression.EStringLiteral sealed interface DataValue : Value @@ -25,4 +26,12 @@ data class RecordValue(val entries: Map>) : DataValue override fun toString(): String { return entries.toString() } + + fun toERecord(): ERecord = ERecord( + entries.mapValues { when(val value = it.value) { + is QuantityValue -> value.toEQuantityScale() + is RecordValue -> value.toERecord() + is StringValue -> value.toEStringLiteral() + } } + ) } diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt index a210b3de..c779a5fa 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt @@ -6,12 +6,15 @@ import ch.kleis.lcaac.core.config.LcaacConfig import ch.kleis.lcaac.core.datasource.in_memory.InMemoryConnector import ch.kleis.lcaac.core.datasource.in_memory.InMemoryConnectorKeys import ch.kleis.lcaac.core.datasource.in_memory.InMemoryDatasource -import ch.kleis.lcaac.core.lang.expression.* +import ch.kleis.lcaac.core.lang.expression.EQuantityScale +import ch.kleis.lcaac.core.lang.expression.ERecord +import ch.kleis.lcaac.core.lang.expression.EStringLiteral +import ch.kleis.lcaac.core.lang.expression.EUnitLiteral import ch.kleis.lcaac.core.lang.fixture.QuantityFixture import ch.kleis.lcaac.core.lang.fixture.QuantityValueFixture import ch.kleis.lcaac.core.lang.fixture.UnitFixture -import ch.kleis.lcaac.core.lang.value.DataSourceValue -import ch.kleis.lcaac.core.lang.value.StringValue +import ch.kleis.lcaac.core.lang.fixture.UnitValueFixture +import ch.kleis.lcaac.core.lang.value.* import ch.kleis.lcaac.core.math.basic.BasicNumber import ch.kleis.lcaac.core.math.basic.BasicOperations import io.mockk.every @@ -34,14 +37,14 @@ class DefaultDataSourceOperationsTest { ) private val ops = BasicOperations - private fun str(s: String): DataExpression = EStringLiteral(s) - private fun numU(value: Double): DataExpression = EQuantityScale( + private fun str(s: String): DataValue = StringValue(s) + private fun numU(value: Double): DataValue = QuantityValue( BasicNumber(value), - UnitFixture.unit, + UnitValueFixture.unit(), ) - private fun numKg(value: Double): DataExpression = EQuantityScale( + private fun numKg(value: Double): DataValue = QuantityValue( BasicNumber(value), - UnitFixture.kg, + UnitValueFixture.kg(), ) @Test @@ -297,7 +300,7 @@ class DefaultDataSourceOperationsTest { "n_items" to numU(1.0), "mass" to numKg(2.0), ), - ).map { ERecord(it) } + ).map { RecordValue(it) } ) val inMemoryConnector = InMemoryConnector( config = InMemoryConnectorKeys.defaultConfig(), @@ -439,7 +442,7 @@ class DefaultDataSourceOperationsTest { "n_items" to numU(1.0), "mass" to numKg(2.0), ), - ).map { ERecord(it) } + ).map { RecordValue(it) } ) val inMemoryConnector = InMemoryConnector( config = InMemoryConnectorKeys.defaultConfig(), @@ -562,7 +565,7 @@ class DefaultDataSourceOperationsTest { "n_items" to numU(1.0), "mass" to numKg(2.0), ), - ).map { ERecord(it) } + ).map { RecordValue(it) } ) val inMemoryConnector = InMemoryConnector( config = InMemoryConnectorKeys.defaultConfig(), @@ -666,4 +669,58 @@ class DefaultDataSourceOperationsTest { ) assertEquals(expected, actual) } + + @Test + fun sumProduct_whenHugeSequence() { + // given + val connector = mockk>() + every { connector.getName() } returns connectorName + every { connector.getConfig() } returns ConnectorConfig( + name = connectorName, + options = emptyMap(), + ) + val n = 5000 + every { connector.getAll(any(), any()) } returns (1..n) + .map { + ERecord(mapOf( + "geo" to EStringLiteral("FR"), + "n_items" to QuantityFixture.oneUnit, + "mass" to QuantityFixture.oneKilogram, + )) + }.asSequence() + + val builder = mockk>() + every { builder.buildOrNull(any(), any()) } returns connector + val factory = ConnectorFactory(".", config, ops, listOf(builder)) + val sourceOps = DefaultDataSourceOperations( + ops, + factory.getLcaacConfig(), + factory.buildConnectors(), + emptyMap() + ) + val source = DataSourceValue( + config = DataSourceConfig( + name = sourceName, + connector = connectorName, + ), + schema = mapOf( + "geo" to StringValue("FR"), + "n_items" to QuantityValueFixture.oneUnit, + "mass" to QuantityValueFixture.oneKilogram, + ), + filter = mapOf( + "geo" to StringValue("FR") + ) + ) + + // when + val actual = sourceOps.sumProduct(source, listOf("n_items", "mass")) + + // then + val expected = EQuantityScale( + BasicNumber(n.toDouble()), + UnitFixture.unit.times(UnitFixture.kg), + ) + assertEquals(expected, actual) + } } diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryConnectorTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryConnectorTest.kt index 642502bc..e3075207 100644 --- a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryConnectorTest.kt +++ b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryConnectorTest.kt @@ -2,29 +2,26 @@ package ch.kleis.lcaac.core.datasource.in_memory import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException -import ch.kleis.lcaac.core.lang.expression.DataExpression -import ch.kleis.lcaac.core.lang.expression.EQuantityScale import ch.kleis.lcaac.core.lang.expression.ERecord import ch.kleis.lcaac.core.lang.expression.EStringLiteral import ch.kleis.lcaac.core.lang.fixture.QuantityFixture import ch.kleis.lcaac.core.lang.fixture.QuantityValueFixture -import ch.kleis.lcaac.core.lang.fixture.UnitFixture -import ch.kleis.lcaac.core.lang.value.DataSourceValue -import ch.kleis.lcaac.core.lang.value.StringValue +import ch.kleis.lcaac.core.lang.fixture.UnitValueFixture +import ch.kleis.lcaac.core.lang.value.* import ch.kleis.lcaac.core.math.basic.BasicNumber import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import kotlin.test.assertEquals class InMemoryConnectorTest { - private fun str(s: String): DataExpression = EStringLiteral(s) - private fun numU(value: Double): DataExpression = EQuantityScale( + private fun str(s: String): DataValue = StringValue(s) + private fun numU(value: Double): DataValue = QuantityValue( BasicNumber(value), - UnitFixture.unit, + UnitValueFixture.unit(), ) - private fun numKg(value: Double): DataExpression = EQuantityScale( + private fun numKg(value: Double): DataValue = QuantityValue( BasicNumber(value), - UnitFixture.kg, + UnitValueFixture.kg(), ) @Test @@ -52,7 +49,7 @@ class InMemoryConnectorTest { "n_items" to numU(1.0), "mass" to numKg(2.0), ), - ).map { ERecord(it) } + ).map { RecordValue(it) } ) val connector = InMemoryConnector( config = InMemoryConnectorKeys.defaultConfig(), @@ -123,7 +120,7 @@ class InMemoryConnectorTest { "n_items" to numU(1.0), "mass" to numKg(2.0), ), - ).map { ERecord(it) } + ).map { RecordValue(it) } ) val connector = InMemoryConnector( config = InMemoryConnectorKeys.defaultConfig(), diff --git a/gradle.properties b/gradle.properties index cb5b64cb..916c2c9a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,4 +2,4 @@ javaVersion=17 gradleVersion=7.6 org.gradle.jvmargs=-Xmx4096m lcaacGroup=ch.kleis.lcaac -lcaacVersion=1.6.8 +lcaacVersion=1.6.9