Skip to content

Commit

Permalink
plugin: basic data source csv loader
Browse files Browse the repository at this point in the history
  • Loading branch information
Peva Blanchard committed Nov 28, 2023
1 parent 88539dc commit d1d41de
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ class DataExpressionReducer<Q>(
private fun reduceDataFrom(expression: EDataFrom<Q>): DataExpression<Q> {
val source = dataSourceRegister[DataSourceKey(expression.source)]
?: throw EvaluatorException("unknown data source ${expression.source}")
return sourceOps.read(source, expression.row, expression.column)
return reduce(sourceOps.read(source, expression.row, expression.column))
}

private fun reduceUnitOf(unitOf: EUnitOf<Q>): DataExpression<Q> {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,49 @@
package ch.kleis.lcaac.plugin.datasources

import ch.kleis.lcaac.core.datasource.DataSourceOperations
import ch.kleis.lcaac.core.lang.expression.DataExpression
import ch.kleis.lcaac.core.lang.expression.DataSourceExpression
import ch.kleis.lcaac.core.lang.expression.SliceIndex
import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException
import ch.kleis.lcaac.core.lang.expression.*
import ch.kleis.lcaac.core.math.QuantityOperations
import org.apache.commons.csv.CSVFormat
import org.apache.commons.csv.CSVParser
import java.nio.file.Path

class LcaDataSourceOperations<Q>(
private val ops: QuantityOperations<Q>,
) : DataSourceOperations<Q> {
override fun read(source: DataSourceExpression<Q>, row: SliceIndex, column: SliceIndex): DataExpression<Q> {
TODO("Not yet implemented")
if (row !is StrIndex || column !is StrIndex) {
throw EvaluatorException("Not supported yet")
}
return when (source) {
is ECsvSource -> readCsv(source, row, column)
}
}

private val format = CSVFormat.DEFAULT.builder()
.setHeader()
.setSkipHeaderRecord(true)
.build()

// TODO: Very inefficient, csv is scanned on every call
private fun readCsv(source: ECsvSource<Q>, row: StrIndex, column: StrIndex): DataExpression<Q> {
val schema = source.schema
val columnType = schema[column.s] ?: throw EvaluatorException("${column.s} not found in schema")

val file = Path.of(source.location).toFile()
val index = source.index ?: throw EvaluatorException("Requires index")
val parser = CSVParser(file.inputStream().reader(), format)
val rawValue = parser.records.find {
it[index] == row.s
}?.get(column.s)
?: throw EvaluatorException("value not found (row=${row.s}, col=${column.s})")

return when(columnType) {
is CQuantity -> {
val amount = ops.pure(rawValue.toDouble())
return EQuantityScale(amount, columnType.unit)
}
is CText -> EStringLiteral(rawValue)
}
}
}

0 comments on commit d1d41de

Please sign in to comment.