From d3cca6903684cd22fd7a954bfc3abe334b453d60 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 29 Jan 2024 16:30:15 +0100 Subject: [PATCH 1/4] cli: trace command --- .../main/kotlin/ch/kleis/lcaac/cli/Main.kt | 2 + .../ch/kleis/lcaac/cli/cmd/TraceCommand.kt | 158 ++++++++++++++++++ .../kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt | 42 ++++- .../ch/kleis/lcaac/cli/csv/CsvProcessor.kt | 41 +---- .../ch/kleis/lcaac/cli/csv/CsvRequest.kt | 4 + .../kotlin/ch/kleis/lcaac/cli/csv/Utils.kt | 10 -- 6 files changed, 207 insertions(+), 50 deletions(-) create mode 100644 cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt delete mode 100644 cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/Utils.kt diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/Main.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/Main.kt index 4637b593..917534cc 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/Main.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/Main.kt @@ -3,11 +3,13 @@ package ch.kleis.lcaac.cli import ch.kleis.lcaac.cli.cmd.AssessCommand import ch.kleis.lcaac.cli.cmd.LcaacCommand import ch.kleis.lcaac.cli.cmd.TestCommand +import ch.kleis.lcaac.cli.cmd.TraceCommand import com.github.ajalt.clikt.core.subcommands fun main(args: Array) = LcaacCommand() .subcommands( AssessCommand(), TestCommand(), + TraceCommand(), ) .main(args) diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt new file mode 100644 index 00000000..f54cc87c --- /dev/null +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt @@ -0,0 +1,158 @@ +package ch.kleis.lcaac.cli.cmd + +import ch.kleis.lcaac.core.assessment.ContributionAnalysisProgram +import ch.kleis.lcaac.core.datasource.CsvSourceOperations +import ch.kleis.lcaac.core.lang.evaluator.Evaluator +import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException +import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer +import ch.kleis.lcaac.core.lang.value.FullyQualifiedSubstanceValue +import ch.kleis.lcaac.core.lang.value.IndicatorValue +import ch.kleis.lcaac.core.lang.value.PartiallyQualifiedSubstanceValue +import ch.kleis.lcaac.core.lang.value.ProductValue +import ch.kleis.lcaac.core.math.basic.BasicOperations +import ch.kleis.lcaac.core.prelude.Prelude.Companion.sanitize +import ch.kleis.lcaac.grammar.Loader +import ch.kleis.lcaac.grammar.LoaderOption +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.arguments.help +import com.github.ajalt.clikt.parameters.options.* +import com.github.ajalt.clikt.parameters.types.file +import org.apache.commons.csv.CSVFormat +import org.apache.commons.csv.CSVPrinter +import java.io.File + +class TraceCommand : CliktCommand(name = "trace", help = "Trace the contributions") { + val name: String by argument().help("Process name") + val labels: Map by option("-l", "--label") + .help( + """ + Specify a process label as a key value pair. + Example: lcaac assess -l model="ABC" -l geo="FR". + """.trimIndent()) + .associate() + private val getPath = option("-p", "--path").file(canBeFile = false).default(File(".")).help("Path to root folder.") + val path: File by getPath + val dataSourcePath: File by option("--data-path").file(canBeFile = false) + .defaultLazy { getPath.value } + .help("Path to data folder. Default to root folder.") + val arguments: Map by option("-D", "--parameter") + .help( + """ + Override parameter value as a key value pair. + Example: `lcaac assess -D x="12 kg" -D geo="UK" -f params.csv`. + """.trimIndent()) + .associate() + + override fun run() { + val ops = BasicOperations + val sourceOps = CsvSourceOperations(dataSourcePath, ops) + val files = lcaFiles(path) + val symbolTable = Loader(ops).load(files, listOf(LoaderOption.WITH_PRELUDE)) + val evaluator = Evaluator(symbolTable, ops, sourceOps) + val template = symbolTable.getTemplate(name, labels) + ?: throw EvaluatorException("unknown template $name$labels") + val args = prepareArguments( + DataExpressionReducer(symbolTable.data, symbolTable.dataSources, ops, sourceOps), + template, + arguments, + ) + val trace = evaluator.trace(template, args) + val system = trace.getSystemValue() + val entryPoint = trace.getEntryPoint() + + val program = ContributionAnalysisProgram(system, entryPoint) + val analysis = program.run() + + val observablePorts = analysis.getObservablePorts() + .getElements() + .sortedWith(trace.getComparator()) + val controllablePorts = analysis.getControllablePorts().getElements() + .sortedBy { it.getShortName() } + + val header = listOf( + "name", "a", "b", "c", "amount", "unit", + ).plus( + controllablePorts.flatMap { + listOf( + sanitize(it.getDisplayName()), + "${sanitize(it.getDisplayName())}_unit" + ) + } + ) + + val lines = observablePorts.asSequence() + .map { row -> + val supply = analysis.supplyOf(row) + val prefix = when (row) { + is IndicatorValue -> { + listOf( + row.name, + "", + "", + "", + supply.amount.toString(), + supply.unit.toString(), + ) + } + + is ProductValue -> { + listOf( + row.name, + row.fromProcessRef?.name ?: "", + row.fromProcessRef?.matchLabels?.toString() ?: "", + row.fromProcessRef?.arguments?.toString() ?: "", + supply.amount.toString(), + supply.unit.toString(), + ) + } + + is FullyQualifiedSubstanceValue -> { + listOf( + row.name, + row.compartment, + row.subcompartment ?: "", + row.type.toString(), + supply.amount.toString(), + supply.unit.toString(), + ) + } + + is PartiallyQualifiedSubstanceValue -> { + listOf( + row.name, + "", + "", + "", + supply.amount.toString(), + supply.unit.toString(), + ) + } + } + val impacts = controllablePorts.flatMap { col -> + val impact = analysis.getPortContribution(row, col) + listOf( + impact.amount.toString(), + impact.unit.toString(), + ) + } + prefix.plus(impacts) + } + + val s = StringBuilder() + CSVPrinter(s, format).printRecord(header) + echo(s.toString(), trailingNewline = false) + lines + .forEach { + s.clear() + CSVPrinter(s, format).printRecord(it) + echo(s.toString(), trailingNewline = false) + } + } + + private val format = CSVFormat.DEFAULT.builder() + .setHeader() + .setSkipHeaderRecord(true) + .setRecordSeparator(System.lineSeparator()) + .build() +} diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt index 143060fb..d202e773 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/Utils.kt @@ -1,8 +1,10 @@ package ch.kleis.lcaac.cli.cmd import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException -import ch.kleis.lcaac.core.lang.expression.DataExpression -import ch.kleis.lcaac.core.lang.expression.EQuantityScale +import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer +import ch.kleis.lcaac.core.lang.expression.* +import ch.kleis.lcaac.core.lang.value.QuantityValue +import ch.kleis.lcaac.core.lang.value.StringValue import ch.kleis.lcaac.core.math.basic.BasicNumber import ch.kleis.lcaac.core.math.basic.BasicOperations import ch.kleis.lcaac.grammar.CoreMapper @@ -55,3 +57,39 @@ fun smartParseQuantityWithDefaultUnit(s: String, defaultUnit: DataExpression, + template: EProcessTemplate, + request: Map, +) = template.params.mapValues { entry -> + when (val v = entry.value) { + is QuantityExpression<*> -> request[entry.key]?.let { + smartParseQuantityWithDefaultUnit(it, EUnitOf(v)) + } ?: entry.value + + is StringExpression -> request[entry.key]?.let { + EStringLiteral(it) + } ?: entry.value + + is EDefaultRecordOf -> { + val dataSource = dataReducer.evalDataSource(v.dataSource) + val schema = dataSource.schema + val entries = schema.mapValues { schemaEntry -> + when (val defaultValue = schemaEntry.value) { + is QuantityValue -> request[schemaEntry.key]?.let { + smartParseQuantityWithDefaultUnit(it, defaultValue.unit.toEUnitLiteral()) + } ?: defaultValue.toEQuantityScale() + + is StringValue -> request[schemaEntry.key]?.let { + EStringLiteral(it) + } ?: defaultValue.toEStringLiteral() + + else -> throw EvaluatorException("datasource '${dataSource.location}': column '${schemaEntry.key}': invalid default value") + } + } + ERecord(entries) + } + + else -> throw EvaluatorException("$v is not a supported data expression") + } +} diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessor.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessor.kt index 0c8c1163..cd4f5e50 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessor.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvProcessor.kt @@ -1,15 +1,12 @@ package ch.kleis.lcaac.cli.csv -import ch.kleis.lcaac.cli.cmd.smartParseQuantityWithDefaultUnit +import ch.kleis.lcaac.cli.cmd.prepareArguments import ch.kleis.lcaac.core.assessment.ContributionAnalysisProgram import ch.kleis.lcaac.core.datasource.CsvSourceOperations import ch.kleis.lcaac.core.lang.SymbolTable import ch.kleis.lcaac.core.lang.evaluator.Evaluator import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException import ch.kleis.lcaac.core.lang.evaluator.reducer.DataExpressionReducer -import ch.kleis.lcaac.core.lang.expression.* -import ch.kleis.lcaac.core.lang.value.QuantityValue -import ch.kleis.lcaac.core.lang.value.StringValue import ch.kleis.lcaac.core.math.basic.BasicNumber import ch.kleis.lcaac.core.math.basic.BasicOperations import java.io.File @@ -28,40 +25,7 @@ class CsvProcessor( val reqLabels = request.matchLabels val template = symbolTable.getTemplate(reqName, reqLabels) ?: throw EvaluatorException("Could not get template for ${reqName}${reqLabels}") - - val arguments = template.params.mapValues { entry -> - when (val v = entry.value) { - is QuantityExpression<*> -> request[entry.key]?.let { - smartParseQuantityWithDefaultUnit(it, EUnitOf(v)) - } ?: entry.value - - is StringExpression -> request[entry.key]?.let { - EStringLiteral(it) - } ?: entry.value - - is EDefaultRecordOf -> { - val dataSource = dataReducer.evalDataSource(v.dataSource) - val schema = dataSource.schema - val entries = schema.mapValues { schemaEntry -> - when (val defaultValue = schemaEntry.value) { - is QuantityValue -> request[schemaEntry.key]?.let { - smartParseQuantityWithDefaultUnit(it, defaultValue.unit.toEUnitLiteral()) - } ?: defaultValue.toEQuantityScale() - - is StringValue -> request[schemaEntry.key]?.let { - EStringLiteral(it) - } ?: defaultValue.toEStringLiteral() - - else -> throw EvaluatorException("datasource '${dataSource.location}': column '${schemaEntry.key}': invalid default value") - } - } - ERecord(entries) - } - - else -> throw EvaluatorException("$v is not a supported data expression") - } - } - + val arguments = prepareArguments(dataReducer, template, request.toMap()) val trace = evaluator.trace(template, arguments) val systemValue = trace.getSystemValue() val entryPoint = trace.getEntryPoint() @@ -77,4 +41,5 @@ class CsvProcessor( ) } } + } diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvRequest.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvRequest.kt index 502e891f..9edb337e 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvRequest.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/CsvRequest.kt @@ -18,4 +18,8 @@ class CsvRequest( return header[name] ?.let { record[it] } } + + fun toMap(): Map { + return header.mapValues { record[it.value] } + } } diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/Utils.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/Utils.kt deleted file mode 100644 index 43a76c28..00000000 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/csv/Utils.kt +++ /dev/null @@ -1,10 +0,0 @@ -package ch.kleis.lcaac.cli.csv - -import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException -import ch.kleis.lcaac.core.lang.expression.DataExpression -import ch.kleis.lcaac.core.lang.expression.EQuantityScale -import ch.kleis.lcaac.core.math.basic.BasicNumber -import java.lang.Double -import kotlin.NumberFormatException -import kotlin.String - From c315debfdd9822f0c582ee3f19607417d158aad3 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 29 Jan 2024 23:32:18 +0100 Subject: [PATCH 2/4] grammar: fix test mapper: parse variables in test --- .../main/kotlin/ch/kleis/lcaac/grammar/CoreTestMapper.kt | 2 ++ .../kotlin/ch/kleis/lcaac/grammar/CoreTestMapperTest.kt | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreTestMapper.kt b/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreTestMapper.kt index 098badaf..baddc619 100644 --- a/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreTestMapper.kt +++ b/grammar/src/main/kotlin/ch/kleis/lcaac/grammar/CoreTestMapper.kt @@ -29,6 +29,8 @@ class CoreTestMapper { } }, template = EProcessTemplate( + locals = ctx.variables().flatMap { it.assignment() } + .associate { it.dataRef().uid().ID().text to coreMapper.dataExpression(it.dataExpression()) }, body = EProcess( name = processName, products = listOf( diff --git a/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreTestMapperTest.kt b/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreTestMapperTest.kt index 1b5e7b70..bd03a397 100644 --- a/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreTestMapperTest.kt +++ b/grammar/src/test/kotlin/ch/kleis/lcaac/grammar/CoreTestMapperTest.kt @@ -13,6 +13,9 @@ class CoreTestMapperTest { // given val content = """ test foo { + variables { + x = 1 kg + } given { 1 kWh electricity } @@ -41,6 +44,9 @@ class CoreTestMapperTest { ) assertEquals( EProcessTemplate( + locals = mapOf( + "x" to EQuantityScale(BasicNumber(1.0), EDataRef("kg")), + ), body = EProcess( name = "__test__foo", products = listOf( From b3a24fbc86f8eef913f4bb4bf392878951303451 Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Mon, 29 Jan 2024 23:32:42 +0100 Subject: [PATCH 3/4] bump version 1.6.1 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 3222fcec..ef759c48 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,4 +2,4 @@ javaVersion=17 gradleVersion=7.6 org.gradle.jvmargs=-Xmx4096m lcaacGroup=ch.kleis.lcaac -lcaacVersion=1.6.0 +lcaacVersion=1.6.1 From 9bb9b784fb5bfbd9d06a2d907e6b4dc6d4fabf7f Mon Sep 17 00:00:00 2001 From: Peva Blanchard Date: Tue, 30 Jan 2024 00:00:41 +0100 Subject: [PATCH 4/4] cli: trace process with multiple products --- cli/README.md | 10 ++ .../ch/kleis/lcaac/cli/cmd/TraceCommand.kt | 129 +++++++++++------- 2 files changed, 89 insertions(+), 50 deletions(-) diff --git a/cli/README.md b/cli/README.md index 31e1e509..fa09a44b 100644 --- a/cli/README.md +++ b/cli/README.md @@ -22,6 +22,16 @@ cd $GIT_ROOT/cli/samples lcaac assess "electricity_mix" --file params.csv ``` +## Tracing + +The `trace` command traces the execution of the assessment of a (single-product) target process. +For every intermediate product or substance, it reports their quantity and their contribution to the total impact. + +```bash +cd $GIT_ROOT/cli/samples +lcaac trace "electricity_mix" +``` + ## Run tests You can run all the tests with the following command. diff --git a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt index f54cc87c..ab0f0e3e 100644 --- a/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt +++ b/cli/src/main/kotlin/ch/kleis/lcaac/cli/cmd/TraceCommand.kt @@ -10,6 +10,7 @@ import ch.kleis.lcaac.core.lang.value.IndicatorValue import ch.kleis.lcaac.core.lang.value.PartiallyQualifiedSubstanceValue import ch.kleis.lcaac.core.lang.value.ProductValue import ch.kleis.lcaac.core.math.basic.BasicOperations +import ch.kleis.lcaac.core.math.basic.BasicOperations.toDouble import ch.kleis.lcaac.core.prelude.Prelude.Companion.sanitize import ch.kleis.lcaac.grammar.Loader import ch.kleis.lcaac.grammar.LoaderOption @@ -22,6 +23,7 @@ import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVPrinter import java.io.File +@Suppress("MemberVisibilityCanBePrivate") class TraceCommand : CliktCommand(name = "trace", help = "Trace the contributions") { val name: String by argument().help("Process name") val labels: Map by option("-l", "--label") @@ -71,6 +73,7 @@ class TraceCommand : CliktCommand(name = "trace", help = "Trace the contribution .sortedBy { it.getShortName() } val header = listOf( + "d_amount", "d_unit", "d_product", "alloc", "name", "a", "b", "c", "amount", "unit", ).plus( controllablePorts.flatMap { @@ -81,63 +84,89 @@ class TraceCommand : CliktCommand(name = "trace", help = "Trace the contribution } ) - val lines = observablePorts.asSequence() - .map { row -> - val supply = analysis.supplyOf(row) - val prefix = when (row) { - is IndicatorValue -> { - listOf( - row.name, - "", - "", - "", - supply.amount.toString(), - supply.unit.toString(), - ) - } + val products = entryPoint.products.asSequence() + val lines = products.flatMap { demandedProduct -> + val demandedAmount = demandedProduct.quantity.amount + val demandedUnit = demandedProduct.quantity.unit + val demandedProductName = demandedProduct.product.name + val allocationAmount = (demandedProduct.allocation?.amount?.toDouble() ?: 1.0) * (demandedProduct.allocation?.unit?.scale ?: 1.0) + observablePorts.asSequence() + .map { row -> + val supply = analysis.supplyOf(row) + val supplyAmount = supply.amount.value * allocationAmount + val prefix = when (row) { + is IndicatorValue -> { + listOf( + demandedAmount.toString(), + demandedUnit.toString(), + demandedProductName, + allocationAmount.toString(), + row.name, + "", + "", + "", + supplyAmount.toString(), + supply.unit.toString(), + ) + } - is ProductValue -> { - listOf( - row.name, - row.fromProcessRef?.name ?: "", - row.fromProcessRef?.matchLabels?.toString() ?: "", - row.fromProcessRef?.arguments?.toString() ?: "", - supply.amount.toString(), - supply.unit.toString(), - ) - } + is ProductValue -> { + listOf( + demandedAmount.toString(), + demandedUnit.toString(), + demandedProductName, + allocationAmount.toString(), + row.name, + row.fromProcessRef?.name ?: "", + row.fromProcessRef?.matchLabels?.toString() ?: "", + row.fromProcessRef?.arguments?.toString() ?: "", + supplyAmount.toString(), + supply.unit.toString(), + ) + } - is FullyQualifiedSubstanceValue -> { - listOf( - row.name, - row.compartment, - row.subcompartment ?: "", - row.type.toString(), - supply.amount.toString(), - supply.unit.toString(), - ) - } + is FullyQualifiedSubstanceValue -> { + listOf( + demandedAmount.toString(), + demandedUnit.toString(), + demandedProductName, + allocationAmount.toString(), + row.name, + row.compartment, + row.subcompartment ?: "", + row.type.toString(), + supplyAmount.toString(), + supply.unit.toString(), + ) + } - is PartiallyQualifiedSubstanceValue -> { + is PartiallyQualifiedSubstanceValue -> { + listOf( + demandedAmount.toString(), + demandedUnit.toString(), + demandedProductName, + allocationAmount.toString(), + row.name, + "", + "", + "", + supplyAmount.toString(), + supply.unit.toString(), + ) + } + } + val impacts = controllablePorts.flatMap { col -> + val impact = analysis.getPortContribution(row, col) + val impactAmount = impact.amount.value * allocationAmount listOf( - row.name, - "", - "", - "", - supply.amount.toString(), - supply.unit.toString(), + impactAmount.toString(), + impact.unit.toString(), ) } + prefix.plus(impacts) } - val impacts = controllablePorts.flatMap { col -> - val impact = analysis.getPortContribution(row, col) - listOf( - impact.amount.toString(), - impact.unit.toString(), - ) - } - prefix.plus(impacts) - } + } + val s = StringBuilder() CSVPrinter(s, format).printRecord(header)