Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feature/rdb user device #31

Merged
merged 5 commits into from
Sep 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import ch.kleis.lcaac.core.datasource.resilio_db.api.RdbClient
import ch.kleis.lcaac.core.datasource.resilio_db.api.SupportedEndpoint
import ch.kleis.lcaac.core.datasource.resilio_db.api.requests.RdbRackServerDeserializer
import ch.kleis.lcaac.core.datasource.resilio_db.api.requests.RdbSwitchDeserializer
import ch.kleis.lcaac.core.datasource.resilio_db.api.requests.RdbUserDeviceDeserializer
import ch.kleis.lcaac.core.lang.SymbolTable
import ch.kleis.lcaac.core.lang.evaluator.EvaluatorException
import ch.kleis.lcaac.core.lang.evaluator.ToValue
Expand All @@ -28,12 +29,14 @@ class ResilioDbConnector<Q>(
private val ops: QuantityOperations<Q>,
url: String,
accessToken: String,
version: String,
private val rdbClientSupplier: (String, LcStepMapping) -> RdbClient<Q> =
{ primaryKey, lcStepMapping ->
RdbClient(
url = url,
accessToken = accessToken,
primaryKey = primaryKey,
version = version,
lcStepMapping = lcStepMapping,
ops = ops,
)
Expand Down Expand Up @@ -107,6 +110,31 @@ class ResilioDbConnector<Q>(
return responses.filter(applyFilter(source.filter))
}

SupportedEndpoint.USER_DEVICE -> {
val deserializer = RdbUserDeviceDeserializer(
options.primaryKey,
ops,
this::localEval,
)
val auxiliaryDataSource = DataSourceValue(
config = auxiliaryDataSourceConfig,
schema = deserializer.schema(),
filter = auxiliaryFilter
)
val auxiliaryRecords = caller.getAll(
auxiliaryDataSource,
)
val requests = auxiliaryRecords.map {
deserializer.deserialize(it)
}
val rdbClient = rdbClientSupplier(options.primaryKey, options.lcStepMapping)
val responses = requests
.flatMap {
rdbClient.userDevice(it)
}
return responses.filter(applyFilter(source.filter))
}

SupportedEndpoint.SWITCH -> {
val deserializer = RdbSwitchDeserializer(
options.primaryKey,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,16 @@ class ResilioDbConnectorBuilder<Q> : ConnectorBuilder<Q> {
?: throw IllegalArgumentException("connector '${ResilioDbConnectorKeys.RDB_CONNECTOR_NAME}': missing option '${ResilioDbConnectorKeys.RDB_URL}'")
val accessToken = config.options[ResilioDbConnectorKeys.RDB_ACCESS_TOKEN]
?: throw IllegalArgumentException("connector '${ResilioDbConnectorKeys.RDB_CONNECTOR_NAME}': missing option '${ResilioDbConnectorKeys.RDB_ACCESS_TOKEN}'")
val version = config.options[ResilioDbConnectorKeys.RDB_VERSION]
?: throw IllegalArgumentException("connector '${ResilioDbConnectorKeys.RDB_CONNECTOR_NAME}': missing option '${ResilioDbConnectorKeys.RDB_VERSION}'")

return ResilioDbConnector(
config = config,
symbolTable = factory.getSymbolTable(),
ops = factory.getQuantityOperations(),
url = url,
accessToken = accessToken,
version = version,
)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,48 +4,9 @@ object ResilioDbConnectorKeys {
const val RDB_CONNECTOR_NAME = "resilio_db"
const val RDB_URL = "url"
const val RDB_ACCESS_TOKEN = "accessToken"
const val RDB_VERSION = "version"

fun requiredOptionKeys() = setOf(
RDB_URL, RDB_ACCESS_TOKEN,
RDB_URL, RDB_ACCESS_TOKEN, RDB_VERSION
)
}

/*
Example lcaac.yaml
--
connectors:
- name: csv
cache:
enabled: true
maxSize: 2048
maxRecordsPerCacheLine: 8192
options:
directory: .
- name: resilio_db
cache:
enabled: true
maxSize: 2048
maxRecordsPerCacheLine: 8192
options:
url: https://db.resilio.tech
accessToken: <secret> # should be read from env var
datasources:
- name: hw_inventory
connector: csv
- name: hw_impacts
connector: resilio_db
options:
# request
primaryKey: id
paramsFrom: hw_inventory
foreignKey: id # hw_impacts will be joined with hw_inventory on pkey = fkey
endpoint: rack_server

# response
lcStepKey: lc_step
manufacturing: manufacturing
transport: transport
use: use
endOfLife: end_of_life
```
*/
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package ch.kleis.lcaac.core.datasource.resilio_db.api

import ch.kleis.lcaac.core.datasource.resilio_db.api.requests.RdbRackServer
import ch.kleis.lcaac.core.datasource.resilio_db.api.requests.RdbSwitch
import ch.kleis.lcaac.core.datasource.resilio_db.api.requests.RdbUserDevice
import ch.kleis.lcaac.core.lang.expression.ERecord
import ch.kleis.lcaac.core.math.QuantityOperations
import com.mayakapps.kache.InMemoryKache
Expand All @@ -15,6 +16,7 @@ import java.net.http.HttpResponse
class RdbClient<Q>(
private val url: String,
private val accessToken: String,
private val version: String,
primaryKey: String,
lcStepMapping: LcStepMapping,
ops: QuantityOperations<Q>,
Expand All @@ -36,6 +38,11 @@ class RdbClient<Q>(
) {
strategy = KacheStrategy.LRU
}
private val userDeviceCache = InMemoryKache<RdbUserDevice, List<ERecord<Q>>>(
maxSize = 1024
) {
strategy = KacheStrategy.LRU
}

fun serverRack(
rdbServerRack: RdbRackServer,
Expand All @@ -61,9 +68,21 @@ class RdbClient<Q>(
?: throw IllegalStateException("rdb client: could not send request $rdbSwitch")
}

fun userDevice(
rdbUserDevice: RdbUserDevice,
): List<ERecord<Q>> {
val result = runBlocking {
userDeviceCache.getOrPut(rdbUserDevice) {
request(rdbUserDevice.id, rdbUserDevice.deviceType.endpoint, rdbUserDevice.json())
}
}
return result
?: throw IllegalStateException("rdb client: could not send request $rdbUserDevice")
}

private fun request(id: String, endpoint: String, requestBody: String): List<ERecord<Q>> {
val request = HttpRequest.newBuilder()
.uri(URI.create("${this.url}/api/${endpoint}"))
.uri(URI.create("${this.url}/api/${endpoint}/${version}"))
.header("Authorization", this.accessToken)
.header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString(requestBody))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ package ch.kleis.lcaac.core.datasource.resilio_db.api

enum class SupportedEndpoint(private val value: String) {
RACK_SERVER("rack_server"),
SWITCH("switch");
SWITCH("switch"),
USER_DEVICE("user_device");

companion object {
fun from(endpoint: String): SupportedEndpoint? = entries
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ class PojoDeserializer<Q>(
}
}

fun readDoubleGb(key: String, entries: Map<String, DataExpression<Q>>): Double {
private fun readDouble(key: String, entries: Map<String, DataExpression<Q>>, unitRef: String): Double {
val data = entries[key]!!
val unit = EDataRef<Q>("GB")
val unit = EDataRef<Q>(unitRef)
val ratio = EQuantityDiv(data, unit)
return when (val value = eval(ratio)) {
is QuantityValue -> with(ops) {
Expand All @@ -44,5 +44,11 @@ class PojoDeserializer<Q>(

else -> throw IllegalArgumentException("${{}.javaClass.name}: invalid record entry '$key': expecting a quantity, found '$value'")
}

}

fun readDoubleGb(key: String, entries: Map<String, DataExpression<Q>>): Double = readDouble(key, entries, "GB")
fun readDoubleWatt(key: String, entries: Map<String, DataExpression<Q>>): Double = readDouble(key, entries, "W")
fun readDoubleHour(key: String, entries: Map<String, DataExpression<Q>>): Double = readDouble(key, entries, "hour")

}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ data class RdbRackServer(
val cpuQuantity: Int,
val ramTotalSizeGb: Double,
val ssdTotalSizeGb: Double,
val usage: RdbUsage,
) {
fun json(): String {
val cpus = (1..cpuQuantity).joinToString(",") {
Expand All @@ -28,7 +29,12 @@ data class RdbRackServer(
],
"ssd_disks": [
{ "size_gb": $ssdTotalSizeGb }
]
],
"usage": {
"geography": "${usage.geography}",
"power_watt": ${usage.powerWatt},
"duration_of_use_hour": ${usage.durationOfUseHour}
}
}
]
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ class RdbRackServerDeserializer<Q>(
private val keyCpuQuantity = "cpu_quantity"
private val keyRamTotalSizeGb = "ram_total_size_gb"
private val keySsdTotalSizeGb = "ssd_total_size_gb"
private val keyGeography = "geography"
private val keyPowerWatt = "power_watt"
private val keyDurationOfUseHour = "duration_of_use_hour"

fun schema(): Map<String, DataValue<Q>> = mapOf(
primaryKey to StringValue("server-01"),
Expand All @@ -31,6 +34,9 @@ class RdbRackServerDeserializer<Q>(
keyCpuQuantity to eval(EDataRef("u")),
keyRamTotalSizeGb to eval(EDataRef("GB")),
keySsdTotalSizeGb to eval(EDataRef("GB")),
keyGeography to StringValue("global"),
keyPowerWatt to eval(EDataRef("W")),
keyDurationOfUseHour to eval(EDataRef("hour")),
)

fun deserialize(record: ERecord<Q>): RdbRackServer {
Expand All @@ -39,6 +45,7 @@ class RdbRackServerDeserializer<Q>(
primaryKey, keyModelName, keyRackUnit,
keyCpuName, keyCpuQuantity,
keyRamTotalSizeGb, keySsdTotalSizeGb,
keyGeography, keyPowerWatt, keyDurationOfUseHour,
)
val missingKeys = requiredKeys.minus(entries.keys)
if (missingKeys.isNotEmpty()) {
Expand All @@ -52,6 +59,11 @@ class RdbRackServerDeserializer<Q>(
cpuQuantity = pojoDeserializer.readIntUnit(keyCpuQuantity, entries),
ramTotalSizeGb = pojoDeserializer.readDoubleGb(keyRamTotalSizeGb, entries),
ssdTotalSizeGb = pojoDeserializer.readDoubleGb(keySsdTotalSizeGb, entries),
usage = RdbUsage(
geography = pojoDeserializer.readString(keyGeography, entries),
powerWatt = pojoDeserializer.readDoubleWatt(keyPowerWatt, entries),
durationOfUseHour = pojoDeserializer.readDoubleHour(keyDurationOfUseHour, entries),
),
)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package ch.kleis.lcaac.core.datasource.resilio_db.api.requests

data class RdbUsage(
val geography: String,
val powerWatt: Double,
val durationOfUseHour: Double,
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package ch.kleis.lcaac.core.datasource.resilio_db.api.requests

enum class RdbUserDeviceType(val endpoint: String) {
SMARTPHONE( "smartphone"),
LAPTOP("laptop"),
DESKTOP( "desktop");

companion object {
fun from(name: String): RdbUserDeviceType {
val names = RdbUserDeviceType.entries.map { it.name }
return RdbUserDeviceType.entries.firstOrNull { it.name.lowercase() == name.lowercase() }
?: throw IllegalArgumentException("invalid device type '$name', available device types are $names")
}
}
}

data class RdbUserDevice(
val id: String,
val deviceType: RdbUserDeviceType,
val modelName: String,
val usage: RdbUsage,
) {
fun json(): String {
return """
{
"assembly": false,
"data": [
{
"wanted_name": "$id",
"name": "$modelName",
"usage": {
"geography": "${usage.geography}",
"power_watt": ${usage.powerWatt},
"duration_of_use_hour": ${usage.durationOfUseHour}
}
}
]
}
""".trimIndent()
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package ch.kleis.lcaac.core.datasource.resilio_db.api.requests

import ch.kleis.lcaac.core.lang.expression.DataExpression
import ch.kleis.lcaac.core.lang.expression.EDataRef
import ch.kleis.lcaac.core.lang.expression.ERecord
import ch.kleis.lcaac.core.lang.value.DataValue
import ch.kleis.lcaac.core.lang.value.StringValue
import ch.kleis.lcaac.core.math.QuantityOperations

class RdbUserDeviceDeserializer<Q>(
private val primaryKey: String,
ops: QuantityOperations<Q>,
private val eval: (DataExpression<Q>) -> DataValue<Q>,
private val pojoDeserializer: PojoDeserializer<Q> = PojoDeserializer(
ops = ops,
eval = eval,
)
) {
private val keyDeviceType = "device_type"
private val keyModelName = "model_name"
private val keyGeography = "geography"
private val keyPowerWatt = "power_watt"
private val keyDurationOfUseHour = "duration_of_use_hour"

fun schema(): Map<String, DataValue<Q>> = mapOf(
primaryKey to StringValue("server-01"),
keyModelName to StringValue("model name"),
keyDeviceType to StringValue("device type"),
keyGeography to StringValue("global"),
keyPowerWatt to eval(EDataRef("W")),
keyDurationOfUseHour to eval(EDataRef("hour")),
)

fun deserialize(record: ERecord<Q>): RdbUserDevice {
val entries = record.entries
val requiredKeys = setOf(
primaryKey, keyModelName, keyDeviceType,
keyGeography, keyPowerWatt, keyDurationOfUseHour,
)
val missingKeys = requiredKeys.minus(entries.keys)
if (missingKeys.isNotEmpty()) {
throw IllegalArgumentException("${{}.javaClass.name}: invalid record: missing keys $missingKeys")
}
return RdbUserDevice(
id = pojoDeserializer.readString(primaryKey, entries),
deviceType = RdbUserDeviceType.from(pojoDeserializer.readString(keyDeviceType, entries)),
modelName = pojoDeserializer.readString(keyModelName, entries),
usage = RdbUsage(
geography = pojoDeserializer.readString(keyGeography, entries),
powerWatt = pojoDeserializer.readDoubleWatt(keyPowerWatt, entries),
durationOfUseHour = pojoDeserializer.readDoubleHour(keyDurationOfUseHour, entries),
),
)
}
}
Loading
Loading