kleis-technology · pevab · Aug 12, 2024 · Aug 12, 2024
diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperations.kt
@@ -139,7 +139,7 @@ class DefaultDataSourceOperations<Q>(
             }.reduce { acc, expression ->
                 reducer.reduce(EQuantityMul(acc, expression))
             }
-        }.fold(zero as DataExpression<Q>, ({ acc, expression -> reducer.reduce(EQuantityAdd(acc, expression)) }))
-
+        }.fold(zero as DataExpression<Q>, ({ acc, expression ->
+            reducer.reduce(EQuantityAdd(acc, expression)) }))
     }
 }
diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryConnector.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryConnector.kt
@@ -5,9 +5,9 @@ import ch.kleis.lcaac.core.config.DataSourceConfig
 import ch.kleis.lcaac.core.datasource.DataSourceConnector
 import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException
 import ch.kleis.lcaac.core.lang.expression.ERecord
-import ch.kleis.lcaac.core.lang.expression.EStringLiteral
 import ch.kleis.lcaac.core.lang.value.DataSourceValue
 import ch.kleis.lcaac.core.lang.value.DataValue
+import ch.kleis.lcaac.core.lang.value.RecordValue
 import ch.kleis.lcaac.core.lang.value.StringValue
 
 class InMemoryConnector<Q>(
@@ -31,7 +31,9 @@ class InMemoryConnector<Q>(
             ?.records
             ?.filter(applyFilter(filter))
             ?: emptyList()
-        return records.asSequence()
+        return records
+            .map { it.toERecord() }
+            .asSequence()
     }
 
     override fun getFirst(config: DataSourceConfig, source: DataSourceValue<Q>): ERecord<Q> {
@@ -42,12 +44,12 @@ class InMemoryConnector<Q>(
 
 private fun <Q> applyFilter(
     filter: Map<String, DataValue<Q>>,
-): (ERecord<Q>) -> Boolean = { record ->
+): (RecordValue<Q>) -> Boolean = { record ->
     filter.entries.all {
         val expected = it.value
         if (expected is StringValue) {
             when (val v = record.entries[it.key]) {
-                is EStringLiteral -> expected.s == v.value
+                is StringValue -> expected.s == v.s
                 else -> throw EvaluatorException("invalid type for column '${it.key}': expected 'EStringLiteral', found '${v?.javaClass?.simpleName}'")
             }
         } else throw EvaluatorException("invalid matching condition $it")

diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryDatasource.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryDatasource.kt
@@ -1,7 +1,7 @@
 package ch.kleis.lcaac.core.datasource.in_memory
 
-import ch.kleis.lcaac.core.lang.expression.ERecord
+import ch.kleis.lcaac.core.lang.value.RecordValue
 
 data class InMemoryDatasource<Q>(
-    val records: List<ERecord<Q>>,
+    val records: List<RecordValue<Q>>,
 )
diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataValue.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/value/DataValue.kt
@@ -1,6 +1,7 @@
 package ch.kleis.lcaac.core.lang.value
 
 import ch.kleis.lcaac.core.lang.expression.EQuantityScale
+import ch.kleis.lcaac.core.lang.expression.ERecord
 import ch.kleis.lcaac.core.lang.expression.EStringLiteral
 
 sealed interface DataValue<Q> : Value<Q>
@@ -25,4 +26,12 @@ data class RecordValue<Q>(val entries: Map<String, DataValue<Q>>) : DataValue<Q>
     override fun toString(): String {
         return entries.toString()
     }
+
+    fun toERecord(): ERecord<Q> = ERecord(
+        entries.mapValues { when(val value = it.value)  {
+            is QuantityValue -> value.toEQuantityScale()
+            is RecordValue -> value.toERecord()
+            is StringValue -> value.toEStringLiteral()
+        } }
+    )
 }
diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/DefaultDataSourceOperationsTest.kt
@@ -6,12 +6,15 @@ import ch.kleis.lcaac.core.config.LcaacConfig
 import ch.kleis.lcaac.core.datasource.in_memory.InMemoryConnector
 import ch.kleis.lcaac.core.datasource.in_memory.InMemoryConnectorKeys
 import ch.kleis.lcaac.core.datasource.in_memory.InMemoryDatasource
-import ch.kleis.lcaac.core.lang.expression.*
+import ch.kleis.lcaac.core.lang.expression.EQuantityScale
+import ch.kleis.lcaac.core.lang.expression.ERecord
+import ch.kleis.lcaac.core.lang.expression.EStringLiteral
+import ch.kleis.lcaac.core.lang.expression.EUnitLiteral
 import ch.kleis.lcaac.core.lang.fixture.QuantityFixture
 import ch.kleis.lcaac.core.lang.fixture.QuantityValueFixture
 import ch.kleis.lcaac.core.lang.fixture.UnitFixture
-import ch.kleis.lcaac.core.lang.value.DataSourceValue
-import ch.kleis.lcaac.core.lang.value.StringValue
+import ch.kleis.lcaac.core.lang.fixture.UnitValueFixture
+import ch.kleis.lcaac.core.lang.value.*
 import ch.kleis.lcaac.core.math.basic.BasicNumber
 import ch.kleis.lcaac.core.math.basic.BasicOperations
 import io.mockk.every
@@ -34,14 +37,14 @@ class DefaultDataSourceOperationsTest {
     )
     private val ops = BasicOperations
 
-    private fun str(s: String): DataExpression<BasicNumber> = EStringLiteral(s)
-    private fun numU(value: Double): DataExpression<BasicNumber> = EQuantityScale(
+    private fun str(s: String): DataValue<BasicNumber> = StringValue(s)
+    private fun numU(value: Double): DataValue<BasicNumber> = QuantityValue(
         BasicNumber(value),
-        UnitFixture.unit,
+        UnitValueFixture.unit(),
     )
-    private fun numKg(value: Double): DataExpression<BasicNumber> = EQuantityScale(
+    private fun numKg(value: Double): DataValue<BasicNumber> = QuantityValue(
         BasicNumber(value),
-        UnitFixture.kg,
+        UnitValueFixture.kg(),
     )
 
     @Test
@@ -297,7 +300,7 @@ class DefaultDataSourceOperationsTest {
                     "n_items" to numU(1.0),
                     "mass" to numKg(2.0),
                 ),
-            ).map { ERecord(it) }
+            ).map { RecordValue(it) }
         )
         val inMemoryConnector = InMemoryConnector(
             config = InMemoryConnectorKeys.defaultConfig(),
@@ -439,7 +442,7 @@ class DefaultDataSourceOperationsTest {
                     "n_items" to numU(1.0),
                     "mass" to numKg(2.0),
                 ),
-            ).map { ERecord(it) }
+            ).map { RecordValue(it) }
         )
         val inMemoryConnector = InMemoryConnector(
             config = InMemoryConnectorKeys.defaultConfig(),
@@ -562,7 +565,7 @@ class DefaultDataSourceOperationsTest {
                     "n_items" to numU(1.0),
                     "mass" to numKg(2.0),
                 ),
-            ).map { ERecord(it) }
+            ).map { RecordValue(it) }
         )
         val inMemoryConnector = InMemoryConnector(
             config = InMemoryConnectorKeys.defaultConfig(),
@@ -666,4 +669,58 @@ class DefaultDataSourceOperationsTest {
         )
         assertEquals(expected, actual)
     }
+
+    @Test
+    fun sumProduct_whenHugeSequence() {
+        // given
+        val connector = mockk<DataSourceConnector<BasicNumber>>()
+        every { connector.getName() } returns connectorName
+        every { connector.getConfig() } returns ConnectorConfig(
+            name = connectorName,
+            options = emptyMap(),
+        )
+        val n = 5000
+        every { connector.getAll(any(), any()) } returns (1..n)
+            .map {
+                ERecord(mapOf(
+                    "geo" to EStringLiteral("FR"),
+                    "n_items" to QuantityFixture.oneUnit,
+                    "mass" to QuantityFixture.oneKilogram,
+                ))
+            }.asSequence()
+
+        val builder = mockk<ConnectorBuilder<BasicNumber>>()
+        every { builder.buildOrNull(any(), any()) } returns connector
+        val factory = ConnectorFactory(".", config, ops, listOf(builder))
+        val sourceOps = DefaultDataSourceOperations(
+            ops,
+            factory.getLcaacConfig(),
+            factory.buildConnectors(),
+            emptyMap()
+        )
+        val source = DataSourceValue(
+            config = DataSourceConfig(
+                name = sourceName,
+                connector = connectorName,
+            ),
+            schema = mapOf(
+                "geo" to StringValue("FR"),
+                "n_items" to QuantityValueFixture.oneUnit,
+                "mass" to QuantityValueFixture.oneKilogram,
+            ),
+            filter = mapOf(
+                "geo" to StringValue("FR")
+            )
+        )
+
+        // when
+        val actual = sourceOps.sumProduct(source, listOf("n_items", "mass"))
+
+        // then
+        val expected = EQuantityScale(
+            BasicNumber(n.toDouble()),
+            UnitFixture.unit.times(UnitFixture.kg),
+        )
+        assertEquals(expected, actual)
+    }
 }
diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryConnectorTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/datasource/in_memory/InMemoryConnectorTest.kt
@@ -2,29 +2,26 @@ package ch.kleis.lcaac.core.datasource.in_memory
 
 import ch.kleis.lcaac.core.config.DataSourceConfig
 import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException
-import ch.kleis.lcaac.core.lang.expression.DataExpression
-import ch.kleis.lcaac.core.lang.expression.EQuantityScale
 import ch.kleis.lcaac.core.lang.expression.ERecord
 import ch.kleis.lcaac.core.lang.expression.EStringLiteral
 import ch.kleis.lcaac.core.lang.fixture.QuantityFixture
 import ch.kleis.lcaac.core.lang.fixture.QuantityValueFixture
-import ch.kleis.lcaac.core.lang.fixture.UnitFixture
-import ch.kleis.lcaac.core.lang.value.DataSourceValue
-import ch.kleis.lcaac.core.lang.value.StringValue
+import ch.kleis.lcaac.core.lang.fixture.UnitValueFixture
+import ch.kleis.lcaac.core.lang.value.*
 import ch.kleis.lcaac.core.math.basic.BasicNumber
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import kotlin.test.assertEquals
 
 class InMemoryConnectorTest {
-    private fun str(s: String): DataExpression<BasicNumber> = EStringLiteral(s)
-    private fun numU(value: Double): DataExpression<BasicNumber> = EQuantityScale(
+    private fun str(s: String): DataValue<BasicNumber> = StringValue(s)
+    private fun numU(value: Double): DataValue<BasicNumber> = QuantityValue(
         BasicNumber(value),
-        UnitFixture.unit,
+        UnitValueFixture.unit(),
     )
-    private fun numKg(value: Double): DataExpression<BasicNumber> = EQuantityScale(
+    private fun numKg(value: Double): DataValue<BasicNumber> = QuantityValue(
         BasicNumber(value),
-        UnitFixture.kg,
+        UnitValueFixture.kg(),
     )
 
     @Test
@@ -52,7 +49,7 @@ class InMemoryConnectorTest {
                     "n_items" to numU(1.0),
                     "mass" to numKg(2.0),
                 ),
-            ).map { ERecord(it) }
+            ).map { RecordValue(it) }
         )
         val connector = InMemoryConnector(
             config = InMemoryConnectorKeys.defaultConfig(),
@@ -123,7 +120,7 @@ class InMemoryConnectorTest {
                     "n_items" to numU(1.0),
                     "mass" to numKg(2.0),
                 ),
-            ).map { ERecord(it) }
+            ).map { RecordValue(it) }
         )
         val connector = InMemoryConnector(
             config = InMemoryConnectorKeys.defaultConfig(),

diff --git a/gradle.properties b/gradle.properties
@@ -2,4 +2,4 @@ javaVersion=17
 gradleVersion=7.6
 org.gradle.jvmargs=-Xmx4096m
 lcaacGroup=ch.kleis.lcaac
-lcaacVersion=1.6.8
+lcaacVersion=1.6.9