Merge pull request #19 from kleis-technology/feature/override

feature/override
kleis-technology · Feb 25, 2024 · a4110a6 · a4110a6
2 parents 6e371ba + bf2ff76
commit a4110a6
Show file tree

Hide file tree

Showing 9 changed files with 146 additions and 83 deletions.
diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperations.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/CsvSourceOperations.kt
@@ -1,14 +1,13 @@
 package ch.kleis.lcaac.core.datasource
 
 import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException
-import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer
-import ch.kleis.lcaac.core.lang.expression.*
-import ch.kleis.lcaac.core.lang.register.DataSourceRegister
-import ch.kleis.lcaac.core.lang.value.DataSourceValue
+import ch.kleis.lcaac.core.lang.expression.DataExpression
+import ch.kleis.lcaac.core.lang.expression.EQuantityScale
+import ch.kleis.lcaac.core.lang.expression.ERecord
+import ch.kleis.lcaac.core.lang.expression.EStringLiteral
 import ch.kleis.lcaac.core.lang.value.QuantityValue
 import ch.kleis.lcaac.core.lang.value.StringValue
 import ch.kleis.lcaac.core.math.QuantityOperations
-import ch.kleis.lcaac.core.prelude.Prelude
 import org.apache.commons.csv.CSVFormat
 import org.apache.commons.csv.CSVParser
 import java.io.File
@@ -23,82 +22,43 @@ class CsvSourceOperations<Q>(
         val location = Paths.get(path.absolutePath, it)
         location.toFile().inputStream()
     }
-) : DataSourceOperations<Q> {
-    private val format = CSVFormat.DEFAULT.builder()
-        .setHeader()
-        .setSkipHeaderRecord(true)
-        .build()
-
-    override fun readAll(source: DataSourceValue<Q>): Sequence<ERecord<Q>> {
-        val inputStream = fileLoader(source.location)
-        val parser = CSVParser(inputStream.reader(), format)
-        val header = parser.headerMap
-        val filter = source.filter
-        return parser.iterator().asSequence()
-            .filter { record ->
-                filter.entries.all {
-                    val iv = it.value
-                    if (iv is StringValue) {
-                        val pos = header[it.key]
-                            ?: throw IllegalStateException(
-                                "${source.location}: invalid schema: unknown column '${it.key}'"
-                            )
-                        record[pos] == iv.s
-                    } else throw EvaluatorException("invalid matching condition $it")
-                }
-            }
-            .map { record ->
-                val entries = header
-                    .filter { entry -> source.schema.containsKey(entry.key) }
-                    .mapValues { entry ->
-                        val columnDefaultValue = source.schema[entry.key]!!
-                        val position = entry.value
-                        val element = record[position]
-                        when (columnDefaultValue) {
-                            is QuantityValue -> parseQuantityWithDefaultUnit(element, columnDefaultValue.unit.toEUnitLiteral())
-                            is StringValue -> EStringLiteral(element)
-                            else -> throw IllegalStateException(
-                                "invalid schema: column '${entry.key}' has an invalid default value"
-                            )
-                        }
+) : DataSourceOperationsBase<Q>(ops, { description ->
+    val inputStream = fileLoader(description.location)
+    val parser = CSVParser(inputStream.reader(), format)
+    val header = parser.headerMap
+    val schema = description.schema
+    parser.iterator().asSequence()
+        .map { csvRecord ->
+            val entries = header
+                .filter { entry -> schema.containsKey(entry.key) }
+                .mapValues { entry ->
+                    val columnDefaultValue = schema[entry.key]!!
+                    val position = entry.value
+                    val element = csvRecord[position]
+                    when (columnDefaultValue) {
+                        is QuantityValue -> parseQuantityWithDefaultUnit(ops, element, columnDefaultValue.unit.toEUnitLiteral())
+                        is StringValue -> EStringLiteral(element)
+                        else -> throw IllegalStateException(
+                            "invalid schema: column '${entry.key}' has an invalid default value"
+                        )
                     }
-                ERecord(entries)
-            }
-    }
-
-    override fun sumProduct(source: DataSourceValue<Q>, columns: List<String>): DataExpression<Q> {
-        val reducer = DataExpressionReducer(
-            dataRegister = Prelude.units(),
-            dataSourceRegister = DataSourceRegister.empty(),
-            ops = ops,
-            sourceOps = this,
-        )
-        return readAll(source).map { record ->
-            columns.map { column ->
-                record.entries[column]
-                    ?: throw IllegalStateException(
-                        "${source.location}: invalid schema: unknown column '$column'"
-                    )
-            }.reduce { acc, expression ->
-                reducer.reduce(EQuantityMul(acc, expression))
-            }
-        }.reduce { acc, expression ->
-            reducer.reduce(EQuantityAdd(acc, expression))
+                }
+            ERecord(entries)
         }
-    }
+})
 
-    private fun parseQuantityWithDefaultUnit(s: String, defaultUnit: DataExpression<Q>):
-        DataExpression<Q> {
-        val amount = try {
-            parseDouble(s)
-        } catch (e: NumberFormatException) {
-            throw EvaluatorException("'$s' is not a valid number")
-        }
-        return EQuantityScale(ops.pure(amount), defaultUnit)
-    }
+private val format = CSVFormat.DEFAULT.builder()
+    .setHeader()
+    .setSkipHeaderRecord(true)
+    .build()
 
-    override fun getFirst(source: DataSourceValue<Q>): ERecord<Q> {
-        return readAll(source).firstOrNull()
-            ?: throw EvaluatorException("no record found in '${source.location}' matching ${source.filter}")
+private fun <Q> parseQuantityWithDefaultUnit(ops: QuantityOperations<Q>, s: String, defaultUnit: DataExpression<Q>):
+    DataExpression<Q> {
+    val amount = try {
+        parseDouble(s)
+    } catch (e: NumberFormatException) {
+        throw EvaluatorException("'$s' is not a valid number")
     }
+    return EQuantityScale(ops.pure(amount), defaultUnit)
 }
+
diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceDescription.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceDescription.kt
@@ -0,0 +1,8 @@
+package ch.kleis.lcaac.core.datasource
+
+import ch.kleis.lcaac.core.lang.value.DataValue
+
+data class DataSourceDescription<Q>(
+    val location: String,
+    val schema: Map<String, DataValue<Q>>,
+)
diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceOperationsBase.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/datasource/DataSourceOperationsBase.kt
@@ -0,0 +1,65 @@
+package ch.kleis.lcaac.core.datasource
+
+import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException
+import ch.kleis.lcaac.core.lang.evaluator.ToValue
+import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer
+import ch.kleis.lcaac.core.lang.expression.DataExpression
+import ch.kleis.lcaac.core.lang.expression.EQuantityAdd
+import ch.kleis.lcaac.core.lang.expression.EQuantityMul
+import ch.kleis.lcaac.core.lang.expression.ERecord
+import ch.kleis.lcaac.core.lang.register.DataSourceRegister
+import ch.kleis.lcaac.core.lang.value.DataSourceValue
+import ch.kleis.lcaac.core.lang.value.StringValue
+import ch.kleis.lcaac.core.math.QuantityOperations
+import ch.kleis.lcaac.core.prelude.Prelude
+
+open class DataSourceOperationsBase<Q>(
+    private val ops: QuantityOperations<Q>,
+    private val load: (DataSourceDescription<Q>) -> Sequence<ERecord<Q>>,
+) : DataSourceOperations<Q> {
+    override fun readAll(source: DataSourceValue<Q>): Sequence<ERecord<Q>> {
+        val description = DataSourceDescription(source.location, source.schema)
+        val records = load(description)
+        val filter = source.filter
+        return records
+            .filter { record ->
+                filter.entries.all {
+                    val expected = it.value
+                    if (expected is StringValue) {
+                        val actual = record.entries[it.key]
+                            ?.let { with(ToValue(ops)) { it.toValue() } }
+                            ?: throw IllegalStateException(
+                                "${source.location}: invalid schema: unknown column '${it.key}'"
+                            )
+                        actual == expected
+                    } else throw EvaluatorException("invalid matching condition $it")
+                }
+            }
+    }
+
+    override fun sumProduct(source: DataSourceValue<Q>, columns: List<String>): DataExpression<Q> {
+        val reducer = DataExpressionReducer(
+            dataRegister = Prelude.units(),
+            dataSourceRegister = DataSourceRegister.empty(),
+            ops = ops,
+            sourceOps = this,
+        )
+        return readAll(source).map { record ->
+            columns.map { column ->
+                record.entries[column]
+                    ?: throw IllegalStateException(
+                        "${source.location}: invalid schema: unknown column '$column'"
+                    )
+            }.reduce { acc, expression ->
+                reducer.reduce(EQuantityMul(acc, expression))
+            }
+        }.reduce { acc, expression ->
+            reducer.reduce(EQuantityAdd(acc, expression))
+        }
+    }
+
+    override fun getFirst(source: DataSourceValue<Q>): ERecord<Q> {
+        return readAll(source).firstOrNull()
+            ?: throw EvaluatorException("no record found in '${source.location}' matching ${source.filter}")
+    }
+}
diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducer.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducer.kt
@@ -50,7 +50,7 @@ class DataExpressionReducer<Q>(
 
     private fun reduceFirstRecordOf(expression: EFirstRecordOf<Q>): DataExpression<Q> {
         val dataSource = evalDataSource(expression.dataSource)
-        return sourceOps.getFirst(dataSource)
+        return reduceMap(sourceOps.getFirst(dataSource))
     }
 
     fun reduceDataSource(expression: DataSourceExpression<Q>, filter: Map<String, DataExpression<Q>> = emptyMap()): EDataSource<Q> {

diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducer.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/LcaExpressionReducer.kt
@@ -85,6 +85,7 @@ class LcaExpressionReducer<Q>(
             is EBlockForEach -> {
                 val ds = dataExpressionReducer.evalDataSource(expression.dataSource)
                 sourceOps.readAll(ds)
+                    .map { dataExpressionReducer.reduce(it) }
                     .flatMap { record ->
                         val reducer = push(mapOf(
                             DataKey(expression.rowRef) to record
@@ -110,6 +111,7 @@ class LcaExpressionReducer<Q>(
             is EBlockForEach -> {
                 val ds = dataExpressionReducer.evalDataSource(expression.dataSource)
                 sourceOps.readAll(ds)
+                    .map { dataExpressionReducer.reduce(it) }
                     .flatMap { record ->
                         val reducer = push(mapOf(
                             DataKey(expression.rowRef) to record
@@ -135,6 +137,7 @@ class LcaExpressionReducer<Q>(
             is EBlockForEach -> {
                 val ds = dataExpressionReducer.evalDataSource(expression.dataSource)
                 sourceOps.readAll(ds)
+                    .map { dataExpressionReducer.reduce(it) }
                     .flatMap { record ->
                         val reducer = push(mapOf(
                             DataKey(expression.rowRef) to record

diff --git a/core/src/main/kotlin/ch/kleis/lcaac/core/lang/register/Register.kt b/core/src/main/kotlin/ch/kleis/lcaac/core/lang/register/Register.kt
@@ -59,5 +59,11 @@ class Register<K, E> (
             data.plus(pairs)
         )
     }
+
+    fun override(key: K, value: E): Register<K, E> {
+        return Register(
+            data.plus(key to value)
+        )
+    }
 }
 
diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/RegisterTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/RegisterTest.kt
@@ -12,6 +12,23 @@ import kotlin.test.assertFailsWith
 import kotlin.test.assertNotEquals
 
 class RegisterTest {
+    @Test
+    fun override() {
+        // given
+        val key = DataKey("abc.x")
+        val a = EDataRef<BasicNumber>("a")
+        val b = EDataRef<BasicNumber>("b")
+        val register = DataRegister<BasicNumber>().plus(mapOf(
+            key to a
+        ))
+
+        // when
+        val actual = register.override(key, b)[key]
+
+        // then
+        assertEquals(b, actual)
+    }
+
     @Test
     fun set_and_get() {
         // given

diff --git a/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducerTest.kt b/core/src/test/kotlin/ch/kleis/lcaac/core/lang/evaluator/reducer/DataExpressionReducerTest.kt
@@ -255,7 +255,7 @@ class DataExpressionReducerTest {
                 "mass" to QuantityFixture.oneKilogram
             )
         )
-        val record = EFirstRecordOf<BasicNumber>(EDataSourceRef("source"))
+        val firstRecordOf = EFirstRecordOf<BasicNumber>(EDataSourceRef("source"))
         val reducer = DataExpressionReducer(
             DataRegister.empty(),
             DataSourceRegister.from(mapOf(
@@ -264,12 +264,16 @@ class DataExpressionReducerTest {
             ops,
             sourceOps,
         )
-        val expected = mockk<ERecord<BasicNumber>>()
+        val expected = ERecord(
+            mapOf(
+                "mass" to QuantityFixture.twoKilograms
+            )
+        )
         val dataSourceValue = with(ToValue(ops)) { dataSource.toValue() }
         every { sourceOps.getFirst(dataSourceValue) } returns expected
 
         // when
-        val actual = reducer.reduce(record)
+        val actual = reducer.reduce(firstRecordOf)
 
         // then
         assertEquals(expected, actual)

diff --git a/gradle.properties b/gradle.properties
@@ -2,4 +2,4 @@ javaVersion=17
 gradleVersion=7.6
 org.gradle.jvmargs=-Xmx4096m
 lcaacGroup=ch.kleis.lcaac
-lcaacVersion=1.6.3
+lcaacVersion=1.6.4