diff --git a/gradle.properties b/gradle.properties index 91b9fbbcf..ffda8672e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,4 +2,4 @@ javaVersion=17 gradleVersion=7.6 org.gradle.jvmargs=-Xmx4096m lcaacGroup=ch.kleis.lcaac -lcaacVersion=1.6.6 +lcaacVersion=1.6.7 diff --git a/plugin/build.gradle.kts b/plugin/build.gradle.kts index 55d7bb2c2..c31aafab7 100644 --- a/plugin/build.gradle.kts +++ b/plugin/build.gradle.kts @@ -47,7 +47,7 @@ sourceSets { } dependencies { - implementation("ch.kleis.lcaac:core:1.6.4") + implementation("ch.kleis.lcaac:core:1.6.5") implementation(files(layout.buildDirectory.dir("stdlib/ef3.1")) { builtBy("generateEmissionFactors31") @@ -72,6 +72,8 @@ dependencies { val kotestVersion = "5.7.2" testImplementation("io.kotest:kotest-runner-junit5-jvm:$kotestVersion") testImplementation("io.kotest:kotest-property-jvm:$kotestVersion") + + implementation("com.charleskorn.kaml:kaml:0.59.0") } diff --git a/plugin/ch.kleis.lcaac.plugin.iml b/plugin/ch.kleis.lcaac.plugin.iml index 24df26b80..85031283b 100644 --- a/plugin/ch.kleis.lcaac.plugin.iml +++ b/plugin/ch.kleis.lcaac.plugin.iml @@ -82,7 +82,8 @@ - + + - + \ No newline at end of file diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ActionHelper.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ActionHelper.kt index ebeb21e92..bb9a742a8 100644 --- a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ActionHelper.kt +++ b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ActionHelper.kt @@ -1,15 +1,15 @@ package ch.kleis.lcaac.plugin.actions -import ch.kleis.lcaac.core.datasource.CsvSourceOperations +import ch.kleis.lcaac.core.datasource.DefaultDataSourceOperations import ch.kleis.lcaac.core.lang.evaluator.EvaluationTrace import ch.kleis.lcaac.core.lang.evaluator.Evaluator import ch.kleis.lcaac.core.math.QuantityOperations +import ch.kleis.lcaac.plugin.ide.config.LcaacConfigExtensions import ch.kleis.lcaac.plugin.language.loader.LcaFileCollector import ch.kleis.lcaac.plugin.language.loader.LcaLoader import ch.kleis.lcaac.plugin.language.psi.LcaFile import com.intellij.openapi.application.runReadAction import com.intellij.openapi.progress.ProgressIndicator -import kotlin.io.path.Path fun traceSystemWithIndicator( indicator: ProgressIndicator, @@ -31,7 +31,12 @@ fun traceSystemWithIndicator( // compute indicator.text = "Solving system" val template = symbolTable.getTemplate(processName, matchLabels)!! // We are called from a process, so it must exist - val projectFile = Path(file.project.basePath!!).toFile() - val sourceOps = CsvSourceOperations(projectFile, ops) + val config = with(LcaacConfigExtensions()) { file.project.lcaacConfig() } + val sourceOps = DefaultDataSourceOperations( + config, + ops, + file.project.basePath!!, + ) + return Evaluator(symbolTable, ops, sourceOps).trace(template) } diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ContributionAnalysisWithDataAction.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ContributionAnalysisWithDataAction.kt index 69024566c..55a7891a0 100644 --- a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ContributionAnalysisWithDataAction.kt +++ b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ContributionAnalysisWithDataAction.kt @@ -40,7 +40,6 @@ class ContributionAnalysisWithDataAction( override fun actionPerformed(e: AnActionEvent) { val project = e.project ?: return - val projectPath = project.basePath?.let { File(it) } ?: return val file = e.getData(LangDataKeys.PSI_FILE) as LcaFile? ?: return val containingDirectory = file.containingDirectory ?: return @@ -62,7 +61,7 @@ class ContributionAnalysisWithDataAction( val parser = LcaLoader(collector.collect(file), BasicOperations) parser.load() } - val csvProcessor = CsvProcessor(projectPath, symbolTable) + val csvProcessor = CsvProcessor(project, symbolTable) val results = requests.flatMap { request -> ProgressManager.checkCanceled() indicator.text = "Processing using ${request.arguments()}" diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessor.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessor.kt index e8c7fd498..b2925da0b 100644 --- a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessor.kt +++ b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessor.kt @@ -1,22 +1,29 @@ package ch.kleis.lcaac.plugin.actions.csv import ch.kleis.lcaac.core.assessment.ContributionAnalysisProgram -import ch.kleis.lcaac.core.datasource.CsvSourceOperations +import ch.kleis.lcaac.core.config.LcaacConfig +import ch.kleis.lcaac.core.datasource.DefaultDataSourceOperations import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.evaluator.Evaluator import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException import ch.kleis.lcaac.core.lang.expression.* import ch.kleis.lcaac.core.math.basic.BasicNumber import ch.kleis.lcaac.core.math.basic.BasicOperations -import java.io.File +import ch.kleis.lcaac.plugin.ide.config.LcaacConfigExtensions +import com.intellij.openapi.project.Project import java.lang.Double.parseDouble class CsvProcessor( - private val rootPath: File, + private val project: Project, private val symbolTable: SymbolTable, + lcaacConfigLoader: () -> LcaacConfig = { with(LcaacConfigExtensions()) { project.lcaacConfig() } }, ) { private val ops = BasicOperations - private val sourceOps = CsvSourceOperations(rootPath, ops) + private val sourceOps = DefaultDataSourceOperations( + lcaacConfigLoader(), + ops, + project.basePath ?: "", + ) private val evaluator = Evaluator(symbolTable, ops, sourceOps) fun process(request: CsvRequest): List { diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/tasks/SensitivityAnalysisTask.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/tasks/SensitivityAnalysisTask.kt index 85f0acf88..a837b372a 100644 --- a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/tasks/SensitivityAnalysisTask.kt +++ b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/tasks/SensitivityAnalysisTask.kt @@ -4,8 +4,8 @@ import arrow.core.filterIsInstance import ch.kleis.lcaac.core.ParameterName import ch.kleis.lcaac.core.assessment.SensitivityAnalysis import ch.kleis.lcaac.core.assessment.SensitivityAnalysisProgram -import ch.kleis.lcaac.core.datasource.CsvSourceOperations import ch.kleis.lcaac.core.datasource.DataSourceOperations +import ch.kleis.lcaac.core.datasource.DefaultDataSourceOperations import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.evaluator.Evaluator import ch.kleis.lcaac.core.lang.evaluator.ToValue @@ -17,6 +17,7 @@ import ch.kleis.lcaac.core.math.dual.DualNumber import ch.kleis.lcaac.core.math.dual.DualOperations import ch.kleis.lcaac.core.matrix.IndexedCollection import ch.kleis.lcaac.core.matrix.ParameterVector +import ch.kleis.lcaac.plugin.ide.config.LcaacConfigExtensions import ch.kleis.lcaac.plugin.language.loader.LcaFileCollector import ch.kleis.lcaac.plugin.language.loader.LcaLoader import ch.kleis.lcaac.plugin.language.psi.LcaFile @@ -32,7 +33,6 @@ import com.intellij.openapi.progress.Task import com.intellij.openapi.project.Project import com.intellij.openapi.wm.ToolWindowManager import com.intellij.ui.content.ContentFactory -import java.io.File class SensitivityAnalysisTask( project: Project, @@ -87,8 +87,12 @@ class SensitivityAnalysisTask( symbolTable.getTemplate( processName, matchLabels - )!! // We are called from a process, so it must exist - val sourceOps = CsvSourceOperations(File(project.basePath!!), ops) + ) ?: throw IllegalStateException("Symbol table: cannot find process '$processName$matchLabels'") + val sourceOps = DefaultDataSourceOperations( + with(LcaacConfigExtensions()) { project.lcaacConfig() }, + ops, + project.basePath ?: throw IllegalStateException("Current project misses a base path"), + ) val (arguments, parameters) = prepareArguments(ops, sourceOps, symbolTable, template.params) val trace = Evaluator(symbolTable, ops, sourceOps).trace(template, arguments) diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/ide/config/LcaacConfigExtensions.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/ide/config/LcaacConfigExtensions.kt new file mode 100644 index 000000000..f8999b84c --- /dev/null +++ b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/ide/config/LcaacConfigExtensions.kt @@ -0,0 +1,34 @@ +package ch.kleis.lcaac.plugin.ide.config + +import ch.kleis.lcaac.core.config.ConnectorConfig +import ch.kleis.lcaac.core.config.LcaacConfig +import com.charleskorn.kaml.Yaml +import com.charleskorn.kaml.decodeFromStream +import com.intellij.openapi.application.runReadAction +import com.intellij.openapi.project.Project +import java.nio.file.Path + +class LcaacConfigExtensions { + fun Project.lcaacConfig(): LcaacConfig { + val workingDirectory = this.basePath ?: throw IllegalStateException("Current project misses a base path") + val candidates = listOf("lcaac.yaml", "lcaac.yml") + return runReadAction { + candidates.firstNotNullOfOrNull { filename -> + val file = Path.of(workingDirectory, filename).toFile() + if (file.exists()) file.inputStream().use { + Yaml.default.decodeFromStream(LcaacConfig.serializer(), it) + } + else null + } ?: LcaacConfig( + connectors = listOf( + ConnectorConfig( + "csv", + options = mapOf( + "directory" to workingDirectory + ) + ) + ) + ) + } + } +} diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/Lca.bnf b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/Lca.bnf index 402195efe..50fab39f4 100644 --- a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/Lca.bnf +++ b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/Lca.bnf @@ -110,7 +110,7 @@ import ::= "import" urn { Data source */ -dataSourceDefinition ::= 'datasource' dataSourceRef '{' (locationField | schemaDefinition)* '}' { +dataSourceDefinition ::= 'datasource' dataSourceRef '{' (locationField | schemaDefinition | block_meta)* '}' { implements=["ch.kleis.lcaac.plugin.language.psi.type.PsiDataSourceDefinition"] mixin="ch.kleis.lcaac.plugin.language.psi.mixin.PsiDataSourceMixin" elementTypeClass="ch.kleis.lcaac.plugin.language.psi.stub.datasource.DataSourceStubElementType" diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/loader/LcaMapper.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/loader/LcaMapper.kt index 485083e36..6cec2ef8f 100644 --- a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/loader/LcaMapper.kt +++ b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/loader/LcaMapper.kt @@ -1,5 +1,6 @@ package ch.kleis.lcaac.plugin.language.loader +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.dimension.Dimension import ch.kleis.lcaac.core.lang.dimension.UnitSymbol @@ -291,13 +292,27 @@ class LcaMapper( } fun dataSourceDefinition(ctx: LcaDataSourceDefinition): EDataSource { + val name = ctx.name val location = ctx.locationFieldList.firstOrNull()?.value?.text?.trim('"') - ?: throw EvaluatorException("missing location field") val schema = ctx.schemaDefinitionList.firstOrNull() ?.columnDefinitionList?.associate { it.getColumnRef().name to dataExpression(it.getValue()) } ?: throw EvaluatorException("missing schema") + val options = ctx.blockMetaList + .flatMap { it.metaAssignmentList } + .associate { + val key = it.getKey() + val value = it.getValue() + key to value + } + val config = DataSourceConfig.completeWithDefaults( + DataSourceConfig( + name = name, + location = location, + options = options, + ) + ) return EDataSource( - location, + config, schema, ) } diff --git a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/testing/LcaTestRunner.kt b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/testing/LcaTestRunner.kt index 629f66faa..db2958625 100644 --- a/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/testing/LcaTestRunner.kt +++ b/plugin/src/main/kotlin/ch/kleis/lcaac/plugin/testing/LcaTestRunner.kt @@ -1,7 +1,7 @@ package ch.kleis.lcaac.plugin.testing import ch.kleis.lcaac.core.assessment.ContributionAnalysisProgram -import ch.kleis.lcaac.core.datasource.CsvSourceOperations +import ch.kleis.lcaac.core.datasource.DefaultDataSourceOperations import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.evaluator.Evaluator import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException @@ -14,6 +14,7 @@ import ch.kleis.lcaac.core.lang.register.Register import ch.kleis.lcaac.core.lang.value.QuantityValueOperations import ch.kleis.lcaac.core.math.basic.BasicNumber import ch.kleis.lcaac.core.math.basic.BasicOperations +import ch.kleis.lcaac.plugin.ide.config.LcaacConfigExtensions import ch.kleis.lcaac.plugin.language.loader.LcaFileCollector import ch.kleis.lcaac.plugin.language.loader.LcaLoader import ch.kleis.lcaac.plugin.language.loader.LcaMapper @@ -23,14 +24,17 @@ import ch.kleis.lcaac.plugin.psi.LcaRangeAssertion import ch.kleis.lcaac.plugin.psi.LcaTest import com.intellij.openapi.application.runReadAction import com.intellij.openapi.project.Project -import java.io.File class LcaTestRunner( private val project: Project, ) { private val ops = BasicOperations private val mapper = LcaMapper(ops) - private val sourceOps = CsvSourceOperations(File(project.basePath!!), ops) + private val sourceOps = DefaultDataSourceOperations( + with(LcaacConfigExtensions()) { project.lcaacConfig() }, + ops, + project.basePath!!, + ) // TODO: Use testing objects from core package fun run(test: LcaTest): LcaTestResult { diff --git a/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessorTest.kt b/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessorTest.kt index 3c2ab058f..c52176eb9 100644 --- a/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessorTest.kt +++ b/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessorTest.kt @@ -1,5 +1,6 @@ package ch.kleis.lcaac.plugin.actions.csv +import ch.kleis.lcaac.core.config.LcaacConfig import ch.kleis.lcaac.core.lang.evaluator.ToValue import ch.kleis.lcaac.core.lang.value.* import ch.kleis.lcaac.core.math.basic.BasicNumber @@ -8,8 +9,10 @@ import ch.kleis.lcaac.core.prelude.Prelude import ch.kleis.lcaac.plugin.fixture.UnitFixture import ch.kleis.lcaac.plugin.language.loader.LcaLoader import ch.kleis.lcaac.plugin.language.psi.LcaFile +import com.intellij.openapi.project.Project import com.intellij.psi.PsiManager import com.intellij.testFramework.fixtures.BasePlatformTestCase +import io.mockk.every import io.mockk.mockk import org.junit.Test import org.junit.runner.RunWith @@ -47,7 +50,9 @@ class CsvProcessorTest : BasePlatformTestCase() { val file = PsiManager.getInstance(project).findFile(vf) as LcaFile val parser = LcaLoader(sequenceOf(file, UnitFixture.getInternalUnitFile(myFixture)), ops) val symbolTable = parser.load() - val processor = CsvProcessor(mockk(), symbolTable) + val project = mockk() + every { project.basePath } returns "working_directory" + val processor = CsvProcessor(project, symbolTable) { LcaacConfig() } val cc = IndicatorValue( "cc", umap["kg"]!!, ) @@ -113,7 +118,9 @@ class CsvProcessorTest : BasePlatformTestCase() { val file = PsiManager.getInstance(project).findFile(vf) as LcaFile val parser = LcaLoader(sequenceOf(file, UnitFixture.getInternalUnitFile(myFixture)), ops) val symbolTable = parser.load() - val csvProcessor = CsvProcessor(mockk(), symbolTable) + val project = mockk() + every { project.basePath } returns "working_directory" + val processor = CsvProcessor(project, symbolTable) { LcaacConfig() } val request = CsvRequest( "p", emptyMap(), @@ -122,7 +129,7 @@ class CsvProcessorTest : BasePlatformTestCase() { ) // when - val actual = csvProcessor.process(request)[0] + val actual = processor.process(request)[0] // then assertEquals(request, actual.request) @@ -178,7 +185,9 @@ class CsvProcessorTest : BasePlatformTestCase() { val file = PsiManager.getInstance(project).findFile(vf) as LcaFile val parser = LcaLoader(sequenceOf(file, UnitFixture.getInternalUnitFile(myFixture)), ops) val symbolTable = parser.load() - val csvProcessor = CsvProcessor(mockk(), symbolTable) + val project = mockk() + every { project.basePath } returns "working_directory" + val processor = CsvProcessor(project, symbolTable) { LcaacConfig() } val request = CsvRequest( "p", mapOf("foo" to "bar"), @@ -187,7 +196,7 @@ class CsvProcessorTest : BasePlatformTestCase() { ) // when - val actual = csvProcessor.process(request)[0] + val actual = processor.process(request)[0] // then assertEquals(request, actual.request) diff --git a/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/e2e/E2ETest.kt b/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/e2e/E2ETest.kt index 42b0d2000..504a37255 100644 --- a/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/e2e/E2ETest.kt +++ b/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/e2e/E2ETest.kt @@ -1,6 +1,7 @@ package ch.kleis.lcaac.plugin.e2e import ch.kleis.lcaac.core.assessment.ContributionAnalysisProgram +import ch.kleis.lcaac.core.config.LcaacConfig import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.dimension.Dimension import ch.kleis.lcaac.core.lang.dimension.UnitSymbol @@ -25,9 +26,11 @@ import ch.kleis.lcaac.plugin.fixture.UnitFixture import ch.kleis.lcaac.plugin.language.loader.LcaFileCollector import ch.kleis.lcaac.plugin.language.loader.LcaLoader import ch.kleis.lcaac.plugin.language.psi.LcaFile +import com.intellij.openapi.project.Project import com.intellij.openapi.vfs.VirtualFile import com.intellij.psi.PsiManager import com.intellij.testFramework.fixtures.BasePlatformTestCase +import io.mockk.every import io.mockk.mockk import junit.framework.TestCase import org.junit.runner.RunWith @@ -370,7 +373,9 @@ class E2ETest : BasePlatformTestCase() { ) val kg = UnitValue(UnitSymbol.of("kg"), 1.0, Dimension.of("mass")) val symbolTable = createFilesAndSymbols(vf) - val csvProcessor = CsvProcessor(mockk(), symbolTable) + val project = mockk() + every { project.basePath } returns "working_directory" + val processor = CsvProcessor(project, symbolTable) { LcaacConfig() } val request = CsvRequest( "p", emptyMap(), @@ -379,7 +384,7 @@ class E2ETest : BasePlatformTestCase() { ) // when - val actual = csvProcessor.process(request) + val actual = processor.process(request) // then TestCase.assertEquals(1, actual.size) diff --git a/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/language/loader/LcaLoaderTest.kt b/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/language/loader/LcaLoaderTest.kt index fe9abcb4c..6b0e0ef57 100644 --- a/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/language/loader/LcaLoaderTest.kt +++ b/plugin/src/test/kotlin/ch/kleis/lcaac/plugin/language/loader/LcaLoaderTest.kt @@ -3,6 +3,7 @@ package ch.kleis.lcaac.plugin.language.loader import arrow.optics.Every import arrow.optics.dsl.index import arrow.optics.typeclasses.Index +import ch.kleis.lcaac.core.config.DataSourceConfig import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException import ch.kleis.lcaac.core.lang.expression.* @@ -12,6 +13,7 @@ import ch.kleis.lcaac.core.math.basic.BasicOperations import ch.kleis.lcaac.plugin.fixture.UnitFixture import ch.kleis.lcaac.plugin.language.psi.LcaFile import com.intellij.testFramework.ParsingTestCase +import io.mockk.mockk import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 @@ -254,7 +256,11 @@ class LcaLoaderTest : ParsingTestCase("", "lca", LcaParserDefinition()) { // then val expected = EDataSource( - "source.csv", + DataSourceConfig( + name = "source", + connector = "csv", + location = "source.csv", + ), mapOf( "geo" to EStringLiteral("GLO"), "mass" to EQuantityScale(BasicNumber(1.0), EDataRef("kg"))