feature/datasources

plugin/build.gradle.kts

@@ -47,7 +47,7 @@ sourceSets {
 }
 
 dependencies {
-    implementation("ch.kleis.lcaac:core:1.3.4")
+    implementation("ch.kleis.lcaac:core:1.6.0")
 
     implementation(files(layout.buildDirectory.dir("stdlib/ef3.1")) {
         builtBy("generateEmissionFactors31")

plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ActionHelper.kt

@@ -1,5 +1,6 @@
 package ch.kleis.lcaac.plugin.actions
 
+import ch.kleis.lcaac.core.datasource.CsvSourceOperations
 import ch.kleis.lcaac.core.lang.evaluator.EvaluationTrace
 import ch.kleis.lcaac.core.lang.evaluator.Evaluator
 import ch.kleis.lcaac.core.math.QuantityOperations
@@ -8,6 +9,7 @@ import ch.kleis.lcaac.plugin.language.loader.LcaLoader
 import ch.kleis.lcaac.plugin.language.psi.LcaFile
 import com.intellij.openapi.application.runReadAction
 import com.intellij.openapi.progress.ProgressIndicator
+import kotlin.io.path.Path
 
 fun <Q> traceSystemWithIndicator(
     indicator: ProgressIndicator,
@@ -29,5 +31,7 @@ fun <Q> traceSystemWithIndicator(
     // compute
     indicator.text = "Solving system"
     val template = symbolTable.getTemplate(processName, matchLabels)!! // We are called from a process, so it must exist
-    return Evaluator(symbolTable, ops).trace(template)
+    val projectFile = Path(file.project.basePath!!).toFile()
+    val sourceOps = CsvSourceOperations(projectFile, ops)
+    return Evaluator(symbolTable, ops, sourceOps).trace(template)
 }

plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/ContributionAnalysisWithDataAction.kt

@@ -20,6 +20,7 @@ import com.intellij.openapi.progress.ProgressIndicator
 import com.intellij.openapi.progress.ProgressManager
 import com.intellij.openapi.progress.Task
 import com.intellij.openapi.vfs.VirtualFileManager
+import java.io.File
 import java.io.FileNotFoundException
 import kotlin.io.path.Path
 
@@ -39,6 +40,7 @@ class ContributionAnalysisWithDataAction(
 
     override fun actionPerformed(e: AnActionEvent) {
         val project = e.project ?: return
+        val projectPath = project.basePath?.let { File(it) } ?: return
         val file = e.getData(LangDataKeys.PSI_FILE) as LcaFile? ?: return
         val containingDirectory = file.containingDirectory ?: return
 
@@ -60,7 +62,7 @@ class ContributionAnalysisWithDataAction(
                         val parser = LcaLoader(collector.collect(file), BasicOperations)
                         parser.load()
                     }
-                    val csvProcessor = CsvProcessor(symbolTable)
+                    val csvProcessor = CsvProcessor(projectPath, symbolTable)
                     val results = requests.flatMap { request ->
                         ProgressManager.checkCanceled()
                         indicator.text = "Processing using ${request.arguments()}"

plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/csv/CsvProcessor.kt

@@ -1,19 +1,23 @@
 package ch.kleis.lcaac.plugin.actions.csv
 
 import ch.kleis.lcaac.core.assessment.ContributionAnalysisProgram
+import ch.kleis.lcaac.core.datasource.CsvSourceOperations
 import ch.kleis.lcaac.core.lang.SymbolTable
 import ch.kleis.lcaac.core.lang.evaluator.Evaluator
 import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException
 import ch.kleis.lcaac.core.lang.expression.*
 import ch.kleis.lcaac.core.math.basic.BasicNumber
 import ch.kleis.lcaac.core.math.basic.BasicOperations
+import java.io.File
 import java.lang.Double.parseDouble
 
 class CsvProcessor(
+    private val rootPath: File,
     private val symbolTable: SymbolTable<BasicNumber>,
 ) {
     private val ops = BasicOperations
-    private val evaluator = Evaluator(symbolTable, ops)
+    private val sourceOps = CsvSourceOperations(rootPath, ops)
+    private val evaluator = Evaluator(symbolTable, ops, sourceOps)
 
     fun process(request: CsvRequest): List<CsvResult> {
         val reqName = request.processName

plugin/src/main/kotlin/ch/kleis/lcaac/plugin/actions/tasks/SensitivityAnalysisTask.kt

@@ -4,6 +4,8 @@ import arrow.core.filterIsInstance
 import ch.kleis.lcaac.core.ParameterName
 import ch.kleis.lcaac.core.assessment.SensitivityAnalysis
 import ch.kleis.lcaac.core.assessment.SensitivityAnalysisProgram
+import ch.kleis.lcaac.core.datasource.CsvSourceOperations
+import ch.kleis.lcaac.core.datasource.DataSourceOperations
 import ch.kleis.lcaac.core.lang.SymbolTable
 import ch.kleis.lcaac.core.lang.evaluator.Evaluator
 import ch.kleis.lcaac.core.lang.evaluator.ToValue
@@ -30,6 +32,7 @@ import com.intellij.openapi.progress.Task
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.wm.ToolWindowManager
 import com.intellij.ui.content.ContentFactory
+import java.io.File
 
 class SensitivityAnalysisTask(
     project: Project,
@@ -85,8 +88,10 @@ class SensitivityAnalysisTask(
                 processName,
                 matchLabels
             )!! // We are called from a process, so it must exist
-        val (arguments, parameters) = prepareArguments(ops, symbolTable, template.params)
-        val trace = Evaluator(symbolTable, ops).trace(template, arguments)
+        val sourceOps = CsvSourceOperations(File(project.basePath!!), ops)
+        val (arguments, parameters) =
+            prepareArguments(ops, sourceOps, symbolTable, template.params)
+        val trace = Evaluator(symbolTable, ops, sourceOps).trace(template, arguments)
         this.analysis = SensitivityAnalysisProgram(trace.getSystemValue(), trace.getEntryPoint(), parameters).run()
     }
 
@@ -123,10 +128,11 @@ class SensitivityAnalysisTask(
 
     private fun prepareArguments(
         ops: DualOperations,
+        sourceOps: DataSourceOperations<DualNumber>,
         symbolTable: SymbolTable<DualNumber>,
         params: Map<String, DataExpression<DualNumber>>
     ): Pair<Map<String, DataExpression<DualNumber>>, ParameterVector<DualNumber>> {
-        val dataReducer = DataExpressionReducer(symbolTable.data, ops)
+        val dataReducer = DataExpressionReducer(symbolTable.data, symbolTable.dataSources, ops, sourceOps)
         val reduced = params.mapValues { dataReducer.reduce(it.value) }
         val quantitativeArgumentList = reduced.filterIsInstance<String, EQuantityScale<DualNumber>>()
             .toList()

plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/Lca.bnf

@@ -51,9 +51,17 @@
       BETWEEN_KEYWORD = 'between'
       AND_KEYWORD = 'and'
 
+      DATASOURCE_KEYWORD = 'datasource'
+      LOCATION_KEYWORD = 'location'
+      SCHEMA_KEYWORD = 'schema'
+      FOR_EACH_KEYWORD = 'for_each'
+      SUM_KEYWORD = 'sum'
+      LOOKUP_KEYWORD = 'lookup'
+      DEFAULT_RECORD_KEYWORD = 'default_record'
+
       NUMBER = 'number'
-      LSQBRACE = ']'
-      RSQBRACE = '['
+      LSQBRACE = '['
+      RSQBRACE = ']'
       LPAREN = '('
       RPAREN = ')'
       LBRACE = '{'
@@ -80,7 +88,7 @@
     LCA File
  */
 
-lcaFile ::= package? import* (process | test | unitDefinition | substance | globalVariables )*
+lcaFile ::= package? import* (process | test | dataSourceDefinition | unitDefinition | substance | globalVariables )*
 
 /*
     Package
@@ -96,6 +104,23 @@ import ::= "import" urn {
     mixin="ch.kleis.lcaac.plugin.language.psi.mixin.trait.PsiUrnOwnerMixin"
 }
 
+/*
+    Data source
+ */
+
+dataSourceDefinition ::= 'datasource' dataSourceRef '{' (locationField | schemaDefinition)* '}' {
+    implements=["ch.kleis.lcaac.plugin.language.psi.type.PsiDataSourceDefinition"]
+    mixin="ch.kleis.lcaac.plugin.language.psi.mixin.PsiDataSourceMixin"
+    elementTypeClass="ch.kleis.lcaac.plugin.language.psi.stub.datasource.DataSourceStubElementType"
+    stubClass = "ch.kleis.lcaac.plugin.language.psi.stub.datasource.DataSourceStub"
+}
+locationField ::= 'location' '=' STRING_LITERAL { methods=[value="STRING_LITERAL"] }
+schemaDefinition ::= 'schema' '{' columnDefinition* '}'
+columnDefinition ::= columnRef '=' dataExpression {
+    implements=["ch.kleis.lcaac.plugin.language.psi.type.PsiColumnDefinition"]
+    mixin="ch.kleis.lcaac.plugin.language.psi.mixin.PsiColumnDefinitionMixin"
+}
+
 /*
     Global variables
  */
@@ -220,7 +245,7 @@ params ::= "params" "{" assignment* "}"
 
 variables ::= "variables" "{" assignment* "}"
 
-assignment ::= dataRef "=" dataExpression {
+assignment ::= dataRef '=' dataExpression{
     implements=["ch.kleis.lcaac.plugin.language.psi.type.PsiAssignment"]
     mixin="ch.kleis.lcaac.plugin.language.psi.mixin.PsiAssignmentMixin"
 }
@@ -245,24 +270,50 @@ block_impacts ::= "impacts" "{" impactExchange* "}"
     Exchanges
  */
 
-technoInputExchange ::= dataExpression inputProductSpec
 technoProductExchange ::= dataExpression outputProductSpec
-bioExchange ::= dataExpression substanceSpec
-impactExchange ::= dataExpression indicatorRef
+
+technoInputExchange ::= terminalTechnoInputExchange
+        | technoBlockForEach
+terminalTechnoInputExchange ::= dataExpression inputProductSpec
+technoBlockForEach ::= 'for_each' dataRef 'from' dataSourceExpression '{' (variables | technoInputExchange)* '}' {
+    implements=["ch.kleis.lcaac.plugin.language.psi.type.PsiBlockForEach"]
+    mixin="ch.kleis.lcaac.plugin.language.psi.mixin.PsiBlockForEachMixin"
+}
+
+bioExchange ::= terminalBioExchange
+        | bioBlockForEach
+terminalBioExchange ::= dataExpression substanceSpec
+bioBlockForEach ::= 'for_each' dataRef 'from' dataSourceExpression '{' (variables | bioExchange)* '}' {
+    implements=["ch.kleis.lcaac.plugin.language.psi.type.PsiBlockForEach"]
+    mixin="ch.kleis.lcaac.plugin.language.psi.mixin.PsiBlockForEachMixin"
+}
+
+impactExchange ::= terminalImpactExchange
+        | impactBlockForEach
+terminalImpactExchange ::= dataExpression indicatorRef
+impactBlockForEach ::= 'for_each' dataRef 'from' dataSourceExpression '{' (variables | impactExchange)* '}' {
+    implements=["ch.kleis.lcaac.plugin.language.psi.type.PsiBlockForEach"]
+    mixin="ch.kleis.lcaac.plugin.language.psi.mixin.PsiBlockForEachMixin"
+}
 
 /*
     Quantity
     Use GrammarKit's expression-related syntax and rules to generate a Pratt parser.
  */
 
-dataExpression ::= addGroup
-           | mulGroup
-           | exponentialQuantityExpression
-           | baseGroup
+dataExpression ::= mulGroup
+    | addGroup
+    | exponentialQuantityExpression
+    | baseGroup
 
 private addGroup ::= addQuantityExpression | subQuantityExpression
 private mulGroup ::= scaleQuantityExpression | mulQuantityExpression | divQuantityExpression
-private baseGroup ::=  parenQuantityExpression | stringExpression | dataRef
+private baseGroup ::=  parenQuantityExpression
+    | stringExpression
+    | recordExpression
+    | colExpression
+    | sliceExpression
+    | dataRef
 
 fake binaryOperatorExpression ::= dataExpression dataExpression {
     extends=dataExpression
@@ -280,6 +331,20 @@ exponentialQuantityExpression ::= dataExpression "^" NUMBER { extends=dataExpres
 parenQuantityExpression ::= "(" dataExpression ")" { extends=dataExpression }
 scaleQuantityExpression ::= NUMBER dataExpression { extends=dataExpression methods=[scale="NUMBER"]}
 stringExpression ::= STRING_LITERAL { extends=dataExpression }
+sliceExpression ::= dataRef  '.' columnRef  { extends=dataExpression }
+recordExpression ::= (opDefaultRecord | opLookup) dataSourceExpression { extends=dataExpression }
+opDefaultRecord ::= 'default_record' 'from'
+opLookup ::= 'lookup'
+colExpression ::= 'sum' '(' dataSourceExpression ',' columnRef ('*' columnRef)* ')' { extends=dataExpression }
+
+/*
+    Data source expression
+ */
+
+dataSourceExpression ::= dataSourceRef rowFilter?
+rowFilter ::= MATCH_KEYWORD rowSelector
+        | MATCH_KEYWORD LPAREN ( rowSelector (COMMA rowSelector)* COMMA? ) RPAREN
+rowSelector ::= columnRef EQUAL dataExpression
 
 
 /*
@@ -336,6 +401,17 @@ testRef ::= uid {
     implements=["ch.kleis.lcaac.plugin.language.psi.type.trait.PsiUIDOwner"]
     mixin="ch.kleis.lcaac.plugin.language.psi.mixin.ref.PsiTestRefMixin"
 }
+
+dataSourceRef ::= uid {
+    implements=["ch.kleis.lcaac.plugin.language.psi.type.ref.PsiDataSourceRef"]
+    mixin="ch.kleis.lcaac.plugin.language.psi.mixin.ref.PsiDataSourceRefMixin"
+}
+
+columnRef ::= uid {
+    implements=["ch.kleis.lcaac.plugin.language.psi.type.ref.PsiColumnRef"]
+    mixin="ch.kleis.lcaac.plugin.language.psi.mixin.ref.PsiColumnRefMixin"
+}
+
 /*
    Spec
  */

plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/Lca.flex

@@ -98,7 +98,13 @@ CommentContent = .*
 <YYINITIAL> "between"                { return LcaTypes.BETWEEN_KEYWORD; }
 <YYINITIAL> "and"                { return LcaTypes.AND_KEYWORD; }
 
-
+<YYINITIAL> "datasource"                { return LcaTypes.DATASOURCE_KEYWORD; }
+<YYINITIAL> "location"                { return LcaTypes.LOCATION_KEYWORD; }
+<YYINITIAL> "schema"                { return LcaTypes.SCHEMA_KEYWORD; }
+<YYINITIAL> "for_each"                { return LcaTypes.FOR_EACH_KEYWORD; }
+<YYINITIAL> "sum"                { return LcaTypes.SUM_KEYWORD; }
+<YYINITIAL> "lookup"                { return LcaTypes.LOOKUP_KEYWORD; }
+<YYINITIAL> "default_record"                { return LcaTypes.DEFAULT_RECORD_KEYWORD; }
 
 <YYINITIAL> [-]?{Number_Int} ("." {Number_Int}? )? {Number_Exp}? { return LcaTypes.NUMBER; }
 <YYINITIAL> {Identifier}             { return LcaTypes.IDENTIFIER; }

plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/ide/insight/LcaBioExchangeAnnotator.kt

@@ -5,31 +5,31 @@ import ch.kleis.lcaac.plugin.language.ide.insight.AnnotatorHelper.annotateWarnWi
 import ch.kleis.lcaac.plugin.language.psi.type.PsiSubstance
 import ch.kleis.lcaac.plugin.language.type_checker.PsiLcaTypeChecker
 import ch.kleis.lcaac.plugin.language.type_checker.PsiTypeCheckException
-import ch.kleis.lcaac.plugin.psi.LcaBioExchange
 import ch.kleis.lcaac.plugin.psi.LcaSubstanceSpec
+import ch.kleis.lcaac.plugin.psi.LcaTerminalBioExchange
 import com.intellij.lang.annotation.AnnotationHolder
 import com.intellij.lang.annotation.Annotator
 import com.intellij.psi.PsiElement
 
 class LcaBioExchangeAnnotator : Annotator {
     override fun annotate(element: PsiElement, holder: AnnotationHolder) {
-        if (element !is LcaBioExchange) {
+        if (element !is LcaTerminalBioExchange) {
             return
         }
 
         checkReferenceResolution(element, holder)
         checkType(element, holder)
     }
 
-    private fun checkReferenceResolution(element: LcaBioExchange, holder: AnnotationHolder) {
+    private fun checkReferenceResolution(element: LcaTerminalBioExchange, holder: AnnotationHolder) {
         val target = element.substanceSpec.reference?.resolve()
         if (target == null || target !is PsiSubstance) {
             val spec = element.substanceSpec
             annotateWarnWithMessage(spec, holder, "unresolved substance ${render(spec)}")
         }
     }
 
-    private fun checkType(element: LcaBioExchange, holder: AnnotationHolder) {
+    private fun checkType(element: LcaTerminalBioExchange, holder: AnnotationHolder) {
         val checker = PsiLcaTypeChecker()
         try {
             checker.check(element)

plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/ide/insight/LcaTechnoInputExchangeAnnotator.kt → plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/ide/insight/LcaTerminalTechnoInputExchangeAnnotator.kt

@@ -8,14 +8,14 @@ import ch.kleis.lcaac.plugin.language.type_checker.PsiLcaTypeChecker
 import ch.kleis.lcaac.plugin.language.type_checker.PsiTypeCheckException
 import ch.kleis.lcaac.plugin.psi.LcaInputProductSpec
 import ch.kleis.lcaac.plugin.psi.LcaOutputProductSpec
-import ch.kleis.lcaac.plugin.psi.LcaTechnoInputExchange
+import ch.kleis.lcaac.plugin.psi.LcaTerminalTechnoInputExchange
 import com.intellij.lang.annotation.AnnotationHolder
 import com.intellij.lang.annotation.Annotator
 import com.intellij.psi.PsiElement
 
-class LcaTechnoInputExchangeAnnotator : Annotator {
+class LcaTerminalTechnoInputExchangeAnnotator : Annotator {
     override fun annotate(element: PsiElement, holder: AnnotationHolder) {
-        if (element !is LcaTechnoInputExchange) {
+        if (element !is LcaTerminalTechnoInputExchange) {
             return
         }
         val targets =

plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/ide/style/LcaFormattingModelBuilder.kt

@@ -85,6 +85,11 @@ class LcaFormattingModelBuilder : FormattingModelBuilder {
                 .spacing(0, 0, 0, true, 1)
                 .beforeInside(ASSERT, TEST)
                 .spacing(0, 0, 0, true, 1)
+                // Datasource
+                .around(DATA_SOURCE_REF).spaces(1)
+                // Block for each
+                .around(MATCH_KEYWORD).spaces(1)
+                .around(FOR_EACH_KEYWORD).spaces(1)
                 // Comments
                 .before(COMMENT_CONTENT)
                 .spaces(0)
@@ -109,6 +114,12 @@ class LcaFormattingModelBuilder : FormattingModelBuilder {
                 .spaces(1)
                 .before(SUBSTANCE_SPEC)
                 .spaces(1)
+                .before(SUM_KEYWORD).spaces(1)
+                .after(SUM_KEYWORD).spaces(0)
+                .around(LOOKUP_KEYWORD).spaces(1)
+                .around(DEFAULT_RECORD_KEYWORD).spaces(1)
+                .before(COMMA).spaces(0)
+                .after(COMMA).spaces(1)
                 // Substances
                 .before(INDICATOR_REF)
                 .spaces(1)

plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/ide/style/LcaIndentBlock.kt

@@ -31,6 +31,7 @@ class LcaIndentBlock(node: ASTNode, private val spaceBuilder: SpacingBuilder) :
             META_ASSIGNMENT, ALIAS_FOR_FIELD,
             COMMENT_LINE, COMMENT_BLOCK_START, COMMENT_CONTENT, COMMENT_BLOCK_END,
             ASSERT, GIVEN, RANGE_ASSERTION,
+            SCHEMA_DEFINITION, COLUMN_DEFINITION, LOCATION_FIELD,
             -> Indent.getNormalIndent(true)
 
             else -> Indent.getNoneIndent()

plugin/src/main/kotlin/ch/kleis/lcaac/plugin/language/ide/syntax/LanguageCompletion.kt

@@ -45,7 +45,10 @@ class LanguageCompletion : CompletionContributor() {
             "reference_unit", "symbol", "dimension", "alias_for", // Unit block
             "variables", "params", "labels", // Process Block
             "products", "inputs", "resources", "emissions", "land_use", "impacts", // Process SubBlocks
-            "test", "given", "assert", "between", "and" // Test blocks
+            "test", "given", "assert", "between", "and", // Test blocks
+            "datasource", "schema", "location", // Data source blocks
+            "for_each", // For each
+            "sum", "lookup", "default_record" // Primitives
         )
     private val listOfKeywordPattern = Regex("(LcaTokenType.*) expected, got")
     private val keywordsPattern = Regex("LcaTokenType\\.([^ ,]*)(, | or |)")