kleis-technology · pevab · Jun 13, 2024 · Jun 10, 2024 · Jun 10, 2024 · Jun 10, 2024
diff --git a/gradle.properties b/gradle.properties
@@ -2,4 +2,4 @@ javaVersion=17
 gradleVersion=7.6
 org.gradle.jvmargs=-Xmx4096m
 lcaacGroup=ch.kleis.lcaac
-lcaacVersion=1.6.6
+lcaacVersion=1.6.7
diff --git a/plugin/build.gradle.kts b/plugin/build.gradle.kts
@@ -47,7 +47,7 @@ sourceSets {
 }
 
 dependencies {
-    implementation("ch.kleis.lcaac:core:1.6.4")
+    implementation("ch.kleis.lcaac:core:1.6.5")
 
     implementation(files(layout.buildDirectory.dir("stdlib/ef3.1")) {
         builtBy("generateEmissionFactors31")
@@ -72,6 +72,8 @@ dependencies {
     val kotestVersion = "5.7.2"
     testImplementation("io.kotest:kotest-runner-junit5-jvm:$kotestVersion")
     testImplementation("io.kotest:kotest-property-jvm:$kotestVersion")
+
+    implementation("com.charleskorn.kaml:kaml:0.59.0")
 }
 
 

diff --git a/plugin/ch.kleis.lcaac.plugin.iml b/plugin/ch.kleis.lcaac.plugin.iml
@@ -82,7 +82,8 @@
     <orderEntry type="library" name="kotlin-reflect" level="project" />
     <orderEntry type="library" name="jetbrains.kotlinx.multik.core.jvm" level="project" />
     <orderEntry type="library" name="jetbrains.kotlinx.serialization.json" level="project" />
-    <orderEntry type="library" name="ch.kleis.lcaac.core" level="project"/>
+    <orderEntry type="library" name="ch.kleis.lcaac.core" level="project" />
     <orderEntry type="library" name="jaxb-api" level="project" />
+    <orderEntry type="library" name="com.charleskorn.kaml" level="project" />
   </component>
-</module>
+</module>
diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ActionHelper.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ActionHelper.kt
@@ -1,15 +1,15 @@
 package ch.kleis.lcaac.plugin.actions
 
-import ch.kleis.lcaac.core.datasource.CsvSourceOperations
+import ch.kleis.lcaac.core.datasource.DefaultDataSourceOperations
 import ch.kleis.lcaac.core.lang.evaluator.EvaluationTrace
 import ch.kleis.lcaac.core.lang.evaluator.Evaluator
 import ch.kleis.lcaac.core.math.QuantityOperations
+import ch.kleis.lcaac.plugin.ide.config.LcaacConfigExtensions
 import ch.kleis.lcaac.plugin.language.loader.LcaFileCollector
 import ch.kleis.lcaac.plugin.language.loader.LcaLoader
 import ch.kleis.lcaac.plugin.language.psi.LcaFile
 import com.intellij.openapi.application.runReadAction
 import com.intellij.openapi.progress.ProgressIndicator
-import kotlin.io.path.Path
 
 fun <Q> traceSystemWithIndicator(
     indicator: ProgressIndicator,
@@ -31,7 +31,12 @@ fun <Q> traceSystemWithIndicator(
     // compute
     indicator.text = "Solving system"
     val template = symbolTable.getTemplate(processName, matchLabels)!! // We are called from a process, so it must exist
-    val projectFile = Path(file.project.basePath!!).toFile()
-    val sourceOps = CsvSourceOperations(projectFile, ops)
+    val config = with(LcaacConfigExtensions()) { file.project.lcaacConfig() }
+    val sourceOps = DefaultDataSourceOperations(
+        config,
+        ops,
+        file.project.basePath!!,
+    )
+
     return Evaluator(symbolTable, ops, sourceOps).trace(template)
 }
diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ContributionAnalysisWithDataAction.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ContributionAnalysisWithDataAction.kt
@@ -40,7 +40,6 @@ class ContributionAnalysisWithDataAction(
 
     override fun actionPerformed(e: AnActionEvent) {
         val project = e.project ?: return
-        val projectPath = project.basePath?.let { File(it) } ?: return
         val file = e.getData(LangDataKeys.PSI_FILE) as LcaFile? ?: return
         val containingDirectory = file.containingDirectory ?: return
 
@@ -62,7 +61,7 @@ class ContributionAnalysisWithDataAction(
                         val parser = LcaLoader(collector.collect(file), BasicOperations)
                         parser.load()
                     }
-                    val csvProcessor = CsvProcessor(projectPath, symbolTable)
+                    val csvProcessor = CsvProcessor(project, symbolTable)
                     val results = requests.flatMap { request ->
                         ProgressManager.checkCanceled()
                         indicator.text = "Processing using ${request.arguments()}"

diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessor.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessor.kt
@@ -1,22 +1,29 @@
 package ch.kleis.lcaac.plugin.actions.csv
 
 import ch.kleis.lcaac.core.assessment.ContributionAnalysisProgram
-import ch.kleis.lcaac.core.datasource.CsvSourceOperations
+import ch.kleis.lcaac.core.config.LcaacConfig
+import ch.kleis.lcaac.core.datasource.DefaultDataSourceOperations
 import ch.kleis.lcaac.core.lang.SymbolTable
 import ch.kleis.lcaac.core.lang.evaluator.Evaluator
 import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException
 import ch.kleis.lcaac.core.lang.expression.*
 import ch.kleis.lcaac.core.math.basic.BasicNumber
 import ch.kleis.lcaac.core.math.basic.BasicOperations
-import java.io.File
+import ch.kleis.lcaac.plugin.ide.config.LcaacConfigExtensions
+import com.intellij.openapi.project.Project
 import java.lang.Double.parseDouble
 
 class CsvProcessor(
-    private val rootPath: File,
+    private val project: Project,
     private val symbolTable: SymbolTable<BasicNumber>,
+    lcaacConfigLoader: () -> LcaacConfig = { with(LcaacConfigExtensions()) { project.lcaacConfig() } },
 ) {
     private val ops = BasicOperations
-    private val sourceOps = CsvSourceOperations(rootPath, ops)
+    private val sourceOps = DefaultDataSourceOperations(
+        lcaacConfigLoader(),
+        ops,
+        project.basePath ?: "",
+    )
     private val evaluator = Evaluator(symbolTable, ops, sourceOps)
 
     fun process(request: CsvRequest): List<CsvResult> {

diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/tasks/SensitivityAnalysisTask.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/tasks/SensitivityAnalysisTask.kt
@@ -4,8 +4,8 @@ import arrow.core.filterIsInstance
 import ch.kleis.lcaac.core.ParameterName
 import ch.kleis.lcaac.core.assessment.SensitivityAnalysis
 import ch.kleis.lcaac.core.assessment.SensitivityAnalysisProgram
-import ch.kleis.lcaac.core.datasource.CsvSourceOperations
 import ch.kleis.lcaac.core.datasource.DataSourceOperations
+import ch.kleis.lcaac.core.datasource.DefaultDataSourceOperations
 import ch.kleis.lcaac.core.lang.SymbolTable
 import ch.kleis.lcaac.core.lang.evaluator.Evaluator
 import ch.kleis.lcaac.core.lang.evaluator.ToValue
@@ -17,6 +17,7 @@ import ch.kleis.lcaac.core.math.dual.DualNumber
 import ch.kleis.lcaac.core.math.dual.DualOperations
 import ch.kleis.lcaac.core.matrix.IndexedCollection
 import ch.kleis.lcaac.core.matrix.ParameterVector
+import ch.kleis.lcaac.plugin.ide.config.LcaacConfigExtensions
 import ch.kleis.lcaac.plugin.language.loader.LcaFileCollector
 import ch.kleis.lcaac.plugin.language.loader.LcaLoader
 import ch.kleis.lcaac.plugin.language.psi.LcaFile
@@ -32,7 +33,6 @@ import com.intellij.openapi.progress.Task
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.wm.ToolWindowManager
 import com.intellij.ui.content.ContentFactory
-import java.io.File
 
 class SensitivityAnalysisTask(
     project: Project,
@@ -87,8 +87,12 @@ class SensitivityAnalysisTask(
             symbolTable.getTemplate(
                 processName,
                 matchLabels
-            )!! // We are called from a process, so it must exist
-        val sourceOps = CsvSourceOperations(File(project.basePath!!), ops)
+            ) ?: throw IllegalStateException("Symbol table: cannot find process '$processName$matchLabels'")
+        val sourceOps = DefaultDataSourceOperations(
+            with(LcaacConfigExtensions()) { project.lcaacConfig() },
+            ops,
+            project.basePath ?: throw IllegalStateException("Current project misses a base path"),
+        )
         val (arguments, parameters) =
             prepareArguments(ops, sourceOps, symbolTable, template.params)
         val trace = Evaluator(symbolTable, ops, sourceOps).trace(template, arguments)

diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/ide/config/LcaacConfigExtensions.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/ide/config/LcaacConfigExtensions.kt
@@ -0,0 +1,34 @@
+package ch.kleis.lcaac.plugin.ide.config
+
+import ch.kleis.lcaac.core.config.ConnectorConfig
+import ch.kleis.lcaac.core.config.LcaacConfig
+import com.charleskorn.kaml.Yaml
+import com.charleskorn.kaml.decodeFromStream
+import com.intellij.openapi.application.runReadAction
+import com.intellij.openapi.project.Project
+import java.nio.file.Path
+
+class LcaacConfigExtensions {
+    fun Project.lcaacConfig(): LcaacConfig {
+        val workingDirectory = this.basePath ?: throw IllegalStateException("Current project misses a base path")
+        val candidates = listOf("lcaac.yaml", "lcaac.yml")
+        return runReadAction {
+            candidates.firstNotNullOfOrNull { filename ->
+                val file = Path.of(workingDirectory, filename).toFile()
+                if (file.exists()) file.inputStream().use {
+                    Yaml.default.decodeFromStream(LcaacConfig.serializer(), it)
+                }
+                else null
+            } ?: LcaacConfig(
+                connectors = listOf(
+                    ConnectorConfig(
+                        "csv",
+                        options = mapOf(
+                            "directory" to workingDirectory
+                        )
+                    )
+                )
+            )
+        }
+    }
+}
diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/Lca.bnf b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/Lca.bnf
@@ -110,7 +110,7 @@ import ::= "import" urn {
     Data source
  */
 
-dataSourceDefinition ::= 'datasource' dataSourceRef '{' (locationField | schemaDefinition)* '}' {
+dataSourceDefinition ::= 'datasource' dataSourceRef '{' (locationField | schemaDefinition | block_meta)* '}' {
     implements=["ch.kleis.lcaac.plugin.language.psi.type.PsiDataSourceDefinition"]
     mixin="ch.kleis.lcaac.plugin.language.psi.mixin.PsiDataSourceMixin"
     elementTypeClass="ch.kleis.lcaac.plugin.language.psi.stub.datasource.DataSourceStubElementType"

diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/loader/LcaMapper.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/loader/LcaMapper.kt
@@ -1,5 +1,6 @@
 package ch.kleis.lcaac.plugin.language.loader
 
+import ch.kleis.lcaac.core.config.DataSourceConfig
 import ch.kleis.lcaac.core.lang.SymbolTable
 import ch.kleis.lcaac.core.lang.dimension.Dimension
 import ch.kleis.lcaac.core.lang.dimension.UnitSymbol
@@ -291,13 +292,27 @@ class LcaMapper<Q>(
     }
 
     fun dataSourceDefinition(ctx: LcaDataSourceDefinition): EDataSource<Q> {
+        val name = ctx.name
         val location = ctx.locationFieldList.firstOrNull()?.value?.text?.trim('"')
-            ?: throw EvaluatorException("missing location field")
         val schema = ctx.schemaDefinitionList.firstOrNull()
             ?.columnDefinitionList?.associate { it.getColumnRef().name to dataExpression(it.getValue()) }
             ?: throw EvaluatorException("missing schema")
+        val options = ctx.blockMetaList
+            .flatMap { it.metaAssignmentList }
+            .associate {
+                val key = it.getKey()
+                val value = it.getValue()
+                key to value
+            }
+        val config = DataSourceConfig.completeWithDefaults(
+            DataSourceConfig(
+                name = name,
+                location = location,
+                options = options,
+            )
+        )
         return EDataSource(
-            location,
+            config,
             schema,
         )
     }

diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/testing/LcaTestRunner.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/testing/LcaTestRunner.kt
@@ -1,7 +1,7 @@
 package ch.kleis.lcaac.plugin.testing
 
 import ch.kleis.lcaac.core.assessment.ContributionAnalysisProgram
-import ch.kleis.lcaac.core.datasource.CsvSourceOperations
+import ch.kleis.lcaac.core.datasource.DefaultDataSourceOperations
 import ch.kleis.lcaac.core.lang.SymbolTable
 import ch.kleis.lcaac.core.lang.evaluator.Evaluator
 import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException
@@ -14,6 +14,7 @@ import ch.kleis.lcaac.core.lang.register.Register
 import ch.kleis.lcaac.core.lang.value.QuantityValueOperations
 import ch.kleis.lcaac.core.math.basic.BasicNumber
 import ch.kleis.lcaac.core.math.basic.BasicOperations
+import ch.kleis.lcaac.plugin.ide.config.LcaacConfigExtensions
 import ch.kleis.lcaac.plugin.language.loader.LcaFileCollector
 import ch.kleis.lcaac.plugin.language.loader.LcaLoader
 import ch.kleis.lcaac.plugin.language.loader.LcaMapper
@@ -23,14 +24,17 @@ import ch.kleis.lcaac.plugin.psi.LcaRangeAssertion
 import ch.kleis.lcaac.plugin.psi.LcaTest
 import com.intellij.openapi.application.runReadAction
 import com.intellij.openapi.project.Project
-import java.io.File
 
 class LcaTestRunner(
     private val project: Project,
 ) {
     private val ops = BasicOperations
     private val mapper = LcaMapper(ops)
-    private val sourceOps = CsvSourceOperations(File(project.basePath!!), ops)
+    private val sourceOps = DefaultDataSourceOperations(
+        with(LcaacConfigExtensions()) { project.lcaacConfig() },
+        ops,
+        project.basePath!!,
+    )
 
     // TODO: Use testing objects from core package
     fun run(test: LcaTest): LcaTestResult {

diff --git a/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessorTest.kt b/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessorTest.kt
@@ -1,5 +1,6 @@
 package ch.kleis.lcaac.plugin.actions.csv
 
+import ch.kleis.lcaac.core.config.LcaacConfig
 import ch.kleis.lcaac.core.lang.evaluator.ToValue
 import ch.kleis.lcaac.core.lang.value.*
 import ch.kleis.lcaac.core.math.basic.BasicNumber
@@ -8,8 +9,10 @@ import ch.kleis.lcaac.core.prelude.Prelude
 import ch.kleis.lcaac.plugin.fixture.UnitFixture
 import ch.kleis.lcaac.plugin.language.loader.LcaLoader
 import ch.kleis.lcaac.plugin.language.psi.LcaFile
+import com.intellij.openapi.project.Project
 import com.intellij.psi.PsiManager
 import com.intellij.testFramework.fixtures.BasePlatformTestCase
+import io.mockk.every
 import io.mockk.mockk
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -47,7 +50,9 @@ class CsvProcessorTest : BasePlatformTestCase() {
         val file = PsiManager.getInstance(project).findFile(vf) as LcaFile
         val parser = LcaLoader(sequenceOf(file, UnitFixture.getInternalUnitFile(myFixture)), ops)
         val symbolTable = parser.load()
-        val processor = CsvProcessor(mockk(), symbolTable)
+        val project = mockk<Project>()
+        every { project.basePath } returns "working_directory"
+        val processor = CsvProcessor(project, symbolTable) { LcaacConfig() }
         val cc = IndicatorValue(
             "cc", umap["kg"]!!,
         )
@@ -113,7 +118,9 @@ class CsvProcessorTest : BasePlatformTestCase() {
         val file = PsiManager.getInstance(project).findFile(vf) as LcaFile
         val parser = LcaLoader(sequenceOf(file, UnitFixture.getInternalUnitFile(myFixture)), ops)
         val symbolTable = parser.load()
-        val csvProcessor = CsvProcessor(mockk(), symbolTable)
+        val project = mockk<Project>()
+        every { project.basePath } returns "working_directory"
+        val processor = CsvProcessor(project, symbolTable) { LcaacConfig() }
         val request = CsvRequest(
             "p",
             emptyMap(),
@@ -122,7 +129,7 @@ class CsvProcessorTest : BasePlatformTestCase() {
         )
 
         // when
-        val actual = csvProcessor.process(request)[0]
+        val actual = processor.process(request)[0]
 
         // then
         assertEquals(request, actual.request)
@@ -178,7 +185,9 @@ class CsvProcessorTest : BasePlatformTestCase() {
         val file = PsiManager.getInstance(project).findFile(vf) as LcaFile
         val parser = LcaLoader(sequenceOf(file, UnitFixture.getInternalUnitFile(myFixture)), ops)
         val symbolTable = parser.load()
-        val csvProcessor = CsvProcessor(mockk(), symbolTable)
+        val project = mockk<Project>()
+        every { project.basePath } returns "working_directory"
+        val processor = CsvProcessor(project, symbolTable) { LcaacConfig() }
         val request = CsvRequest(
             "p",
             mapOf("foo" to "bar"),
@@ -187,7 +196,7 @@ class CsvProcessorTest : BasePlatformTestCase() {
         )
 
         // when
-        val actual = csvProcessor.process(request)[0]
+        val actual = processor.process(request)[0]
 
         // then
         assertEquals(request, actual.request)

diff --git a/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/e2e/E2ETest.kt b/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/e2e/E2ETest.kt
@@ -1,6 +1,7 @@
 package ch.kleis.lcaac.plugin.e2e
 
 import ch.kleis.lcaac.core.assessment.ContributionAnalysisProgram
+import ch.kleis.lcaac.core.config.LcaacConfig
 import ch.kleis.lcaac.core.lang.SymbolTable
 import ch.kleis.lcaac.core.lang.dimension.Dimension
 import ch.kleis.lcaac.core.lang.dimension.UnitSymbol
@@ -25,9 +26,11 @@ import ch.kleis.lcaac.plugin.fixture.UnitFixture
 import ch.kleis.lcaac.plugin.language.loader.LcaFileCollector
 import ch.kleis.lcaac.plugin.language.loader.LcaLoader
 import ch.kleis.lcaac.plugin.language.psi.LcaFile
+import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.psi.PsiManager
 import com.intellij.testFramework.fixtures.BasePlatformTestCase
+import io.mockk.every
 import io.mockk.mockk
 import junit.framework.TestCase
 import org.junit.runner.RunWith
@@ -370,7 +373,9 @@ class E2ETest : BasePlatformTestCase() {
         )
         val kg = UnitValue<BasicNumber>(UnitSymbol.of("kg"), 1.0, Dimension.of("mass"))
         val symbolTable = createFilesAndSymbols(vf)
-        val csvProcessor = CsvProcessor(mockk(), symbolTable)
+        val project = mockk<Project>()
+        every { project.basePath } returns "working_directory"
+        val processor = CsvProcessor(project, symbolTable) { LcaacConfig() }
         val request = CsvRequest(
             "p",
             emptyMap(),
@@ -379,7 +384,7 @@ class E2ETest : BasePlatformTestCase() {
         )
 
         // when
-        val actual = csvProcessor.process(request)
+        val actual = processor.process(request)
 
         // then
         TestCase.assertEquals(1, actual.size)