Skip to content

Commit

Permalink
Feature/fix missing agranimo data import (#121)
Browse files Browse the repository at this point in the history
  • Loading branch information
@saschadoemer
saschadoemer authored Jan 13, 2024
1 parent 5e713de commit 692006a
Show file tree
Hide file tree
Showing 20 changed files with 296 additions and 106 deletions.
3 changes: 2 additions & 1 deletion src/main/java/de/app/fivegla/api/Error.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ public enum Error {
COULD_NOT_PARSE_GEO_JSON(errorOf(27)),
COULD_NOT_PARSE_CSV(errorOf(28)),
THIRD_PARTY_SERVICE_UNAVAILABLE(errorOf(29)),
MICASENSE_TX_NOT_FOUND(errorOf(30));
MICASENSE_TX_NOT_FOUND(errorOf(30)),
AGRANIMO_COULD_NOT_FETCH_SOIL_MOISTURE(errorOf(31));

private static String errorOf(int i) {
return ERR_ + String.format("%05d", i);
Expand Down
74 changes: 71 additions & 3 deletions ...ain/java/de/app/fivegla/integration/agranimo/AgranimoFiwareIntegrationServiceWrapper.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,19 @@
package de.app.fivegla.integration.agranimo;


import de.app.fivegla.api.FiwareDevicMeasurementeId;
import de.app.fivegla.api.FiwareDeviceId;
import de.app.fivegla.api.Format;
import de.app.fivegla.config.ApplicationConfiguration;
import de.app.fivegla.config.manufacturer.CommonManufacturerConfiguration;
import de.app.fivegla.fiware.DeviceIntegrationService;
import de.app.fivegla.fiware.DeviceMeasurementIntegrationService;
import de.app.fivegla.integration.soilscout.model.SensorData;
import de.app.fivegla.fiware.model.Device;
import de.app.fivegla.fiware.model.DeviceCategory;
import de.app.fivegla.fiware.model.DeviceMeasurement;
import de.app.fivegla.fiware.model.Location;
import de.app.fivegla.integration.agranimo.model.SoilMoisture;
import de.app.fivegla.integration.agranimo.model.Zone;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
Expand All @@ -18,10 +28,68 @@
@RequiredArgsConstructor
@SuppressWarnings({"FieldCanBeLocal", "unused"})
public class AgranimoFiwareIntegrationServiceWrapper {
private final ApplicationConfiguration applicationConfiguration;
private final DeviceIntegrationService deviceIntegrationService;
private final DeviceMeasurementIntegrationService deviceMeasurementIntegrationService;

public void persist(List<SensorData> measurements) {
log.error("Not implemented yet");
/**
* Persists the soil moisture measurement for a given zone.
*
* @param zone the zone associated with the soil moisture measurement
* @param soilMoisture the soil moisture measurement to persist
*/
public void persist(Zone zone, SoilMoisture soilMoisture) {
persist(soilMoisture.getDeviceId(), zone.getData().getPoint().getCoordinates()[0], zone.getData().getPoint().getCoordinates()[1]);
var deviceMeasurement = createDeviceMeasurements(zone, soilMoisture);
log.info("Persisting measurement for device: {}", soilMoisture.getDeviceId());
deviceMeasurementIntegrationService.persist(deviceMeasurement);
}

private void persist(String deviceId, double latitude, double longitude) {
var device = Device.builder()
.id(FiwareDeviceId.create(getManufacturerConfiguration(), deviceId))
.manufacturerSpecificId(deviceId)
.deviceCategory(DeviceCategory.builder()
.value(List.of(getManufacturerConfiguration().getKey()))
.build())
.location(Location.builder()
.coordinates(List.of(latitude, longitude))
.build())
.build();
deviceIntegrationService.persist(device);
}

private DeviceMeasurement createDeviceMeasurements(Zone zone, SoilMoisture soilMoisture) {
log.debug("Persisting data for zone: {}", zone.getId());
return DeviceMeasurement.builder()
.id(FiwareDevicMeasurementeId.create(getManufacturerConfiguration()))
.refDevice(FiwareDeviceId.create(getManufacturerConfiguration(), String.valueOf(soilMoisture.getDeviceId())))
.dateObserved(Format.format(soilMoisture.getTms()))
.location(Location.builder()
.coordinates(List.of(zone.getData().getPoint().getCoordinates()[0], zone.getData().getPoint().getCoordinates()[1]))
.build())
.controlledProperty("smo1")
.numValue(soilMoisture.getSmo1())
.controlledProperty("smo2")
.numValue(soilMoisture.getSmo2())
.controlledProperty("smo3")
.numValue(soilMoisture.getSmo3())
.controlledProperty("smo4")
.numValue(soilMoisture.getSmo4())
.build();
}

private CommonManufacturerConfiguration getManufacturerConfiguration() {
return applicationConfiguration.getSensors().agranimo();
}

/**
* Retrieves the unique device ID of the specified device.
*
* @param deviceId The unique identifier of the device.
* @return The device ID.
*/
public String deviceIdOf(String deviceId) {
return FiwareDeviceId.create(getManufacturerConfiguration(), deviceId);
}
}
4 changes: 2 additions & 2 deletions ...ion/agranimo/LoginIntegrationService.java → ...nimo/AgranimoLoginIntegrationService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import de.app.fivegla.api.ErrorMessage;
import de.app.fivegla.api.exceptions.BusinessException;
import de.app.fivegla.integration.agranimo.cache.UserDataCache;
import de.app.fivegla.integration.agranimo.dto.Credentials;
import de.app.fivegla.integration.agranimo.model.Credentials;
import de.app.fivegla.integration.agranimo.dto.request.LoginRequest;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -19,7 +19,7 @@
@Slf4j
@Service
@RequiredArgsConstructor
public class LoginIntegrationService {
public class AgranimoLoginIntegrationService {

@Value("${app.sensors.agranimo.url}")
private String url;
Expand Down
44 changes: 40 additions & 4 deletions src/main/java/de/app/fivegla/integration/agranimo/AgranimoMeasurementImport.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
package de.app.fivegla.integration.agranimo;

import de.app.fivegla.api.Manufacturer;
import de.app.fivegla.integration.agranimo.model.SoilMoisture;
import de.app.fivegla.integration.agranimo.model.Zone;
import de.app.fivegla.monitoring.JobMonitor;
import de.app.fivegla.persistence.ApplicationDataRepository;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;

Expand All @@ -20,18 +23,42 @@
public class AgranimoMeasurementImport {

private final ApplicationDataRepository applicationDataRepository;
@SuppressWarnings({"FieldCanBeLocal", "unused"})
private final AgranimoFiwareIntegrationServiceWrapper fiwareIntegrationServiceWrapper;
private final AgranimoSoilMoistureIntegrationService agranimoSoilMoistureIntegrationService;
private final AgranimoZoneService agranimoZoneService;
private final JobMonitor jobMonitor;

@Value("${app.scheduled.daysInThePastForInitialImport}")
private int daysInThePastForInitialImport;

@Async
public void run() {
var begin = Instant.now();
try {
if (applicationDataRepository.getLastRun(Manufacturer.AGRANIMO).isPresent()) {
jobMonitor.logNrOfEntitiesFetched(Manufacturer.AGRANIMO, 0);
var lastRun = applicationDataRepository.getLastRun(Manufacturer.AGRANIMO);
if (lastRun.isPresent()) {
log.info("Running scheduled data import from Agranimo API");
agranimoZoneService.fetchZones().forEach(zone -> {
var waterContent = agranimoSoilMoistureIntegrationService.fetchWaterContent(zone, lastRun.get());
jobMonitor.logNrOfEntitiesFetched(Manufacturer.AGRANIMO, waterContent.size());
log.info("Found {} water content entries", waterContent.size());
log.info("Persisting {} water content entries", waterContent.size());
waterContent.forEach(
soilMoisture -> persistDataWithinFiware(zone, soilMoisture)
);
});

} else {
jobMonitor.logNrOfEntitiesFetched(Manufacturer.AGRANIMO, 0);
log.info("Running scheduled data import from Agranimo API");
agranimoZoneService.fetchZones().forEach(zone -> {
var waterContent = agranimoSoilMoistureIntegrationService.fetchWaterContent(zone, Instant.now().minus(daysInThePastForInitialImport, ChronoUnit.DAYS));
jobMonitor.logNrOfEntitiesFetched(Manufacturer.AGRANIMO, waterContent.size());
log.info("Found {} water content entries", waterContent.size());
log.info("Persisting {} water content entries", waterContent.size());
waterContent.forEach(
soilMoisture -> persistDataWithinFiware(zone, soilMoisture)
);
});
}
applicationDataRepository.updateLastRun(Manufacturer.AGRANIMO);
} catch (Exception e) {
Expand All @@ -44,4 +71,13 @@ public void run() {
}
}

private void persistDataWithinFiware(Zone zone, SoilMoisture soilMoisture) {
try {
fiwareIntegrationServiceWrapper.persist(zone, soilMoisture);
} catch (Exception e) {
log.error("Error while running scheduled data import from Agranimo API", e);
jobMonitor.logErrorDuringExecution(Manufacturer.AGRANIMO);
}
}

}
4 changes: 2 additions & 2 deletions src/main/java/de/app/fivegla/integration/agranimo/AgranimoMonitoring.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
public class AgranimoMonitoring {

private final ApplicationConfiguration applicationConfiguration;
private final ZoneService zoneService;
private final AgranimoZoneService agranimoZoneService;

@ReadOperation
public Health read() {
Expand All @@ -32,7 +32,7 @@ public Health read() {
return null;
} else {
try {
var zones = zoneService.fetchZones();
var zones = agranimoZoneService.fetchZones();
if (zones != null && !zones.isEmpty()) {
return Health
.up()
Expand Down
119 changes: 61 additions & 58 deletions ...main/java/de/app/fivegla/integration/agranimo/AgranimoSoilMoistureIntegrationService.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
package de.app.fivegla.integration.agranimo;

import de.app.fivegla.api.Error;
import de.app.fivegla.api.ErrorMessage;
import de.app.fivegla.api.exceptions.BusinessException;
import de.app.fivegla.integration.agranimo.model.SoilMoisture;
import de.app.fivegla.integration.agranimo.model.SoilMoistureType;
import de.app.fivegla.integration.agranimo.model.Zone;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
Expand All @@ -11,6 +16,7 @@

import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Map;

/**
Expand All @@ -24,80 +30,77 @@ public class AgranimoSoilMoistureIntegrationService {
@Value("${app.sensors.agranimo.url}")
private String url;

private final ZoneService zoneService;
private final LoginIntegrationService loginService;
private final AgranimoLoginIntegrationService loginService;
private final RestTemplate restTemplate;

/**
* Fetch the water content from the API.
*
* @param zone The zone to fetch the data for.
* @param since The date since to fetch the data.
* @param until The date until to fetch the data.
* @return The water content.
*/
public void fetchWaterContent(Instant since, Instant until) {
fetchAll(since, until, SoilMoistureType.WATER_CONTENT);
}

/**
* Fetch the water height from the API.
*
* @param since The date since to fetch the data.
* @param until The date until to fetch the data.
*/
public void fetchWaterHeight(Instant since, Instant until) {
fetchAll(since, until, SoilMoistureType.WATER_HEIGHT);
}

/**
* Fetch the water volume from the API.
*
* @param since The date since to fetch the data.
* @param until The date until to fetch the data.
*/
public void fetchWaterVolume(Instant since, Instant until) {
fetchAll(since, until, SoilMoistureType.WATER_VOLUMETRIC);
public List<SoilMoisture> fetchWaterContent(Zone zone, Instant since) {
return fetchAll(zone, since);
}

/**
* Fetch the soil moisture from the API.
*/
private void fetchAll(Instant since, Instant until, SoilMoistureType soilMoistureType) {
zoneService.fetchZones().forEach(zone -> {
log.info("Fetching soil moisture data for zone {}.", zone.getName());
log.debug("Fetching soil moisture data for zone {} from {} to {}.", zone.getId(), since, until);
try {
var headers = new HttpHeaders();
headers.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
headers.setBearerAuth(loginService.fetchAccessToken());
var httpEntity = new HttpEntity<>(headers);
var uri = UriComponentsBuilder.fromHttpUrl(url + "/zone/{zoneId}/data/soilmoisture?dateStart={since}&dateEnd={until}&type={type}")
.encode()
.toUriString();
var uriVariables = Map.of(
"zoneId",
zone.getId(),
"since",
since.getEpochSecond(),
"until",
until.getEpochSecond(),
"type",
soilMoistureType.getKey());
log.debug("Calling API with URI {} and variables {}.", uri, uriVariables);
var response = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, String.class, uriVariables);
private List<SoilMoisture> fetchAll(Zone zone, Instant since) {
var until = Instant.now();
log.info("Fetching soil moisture data for zone {}.", zone.getName());
log.debug("Fetching soil moisture data for zone {} from {} to {}.", zone.getId(), since, until);
try {
var headers = new HttpHeaders();
headers.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
headers.setBearerAuth(loginService.fetchAccessToken());
var httpEntity = new HttpEntity<>(headers);
var uri = UriComponentsBuilder.fromHttpUrl(url + "/zone/{zoneId}/data/soilmoisture?dateStart={since}&dateEnd={until}&type={type}")
.encode()
.toUriString();
var uriVariables = Map.of(
"zoneId",
zone.getId(),
"since",
since.getEpochSecond(),
"until",
until.getEpochSecond(),
"type",
SoilMoistureType.WATER_CONTENT.getKey());
log.debug("Calling API with URI {} and variables {}.", uri, uriVariables);
var response = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, SoilMoisture[].class, uriVariables);

if (response.getStatusCode() != HttpStatus.OK) {
log.error("Error while fetching soil moisture from the API. Status code: {}", response.getStatusCode());
log.info("Could not fetch soil moisture data for zone {}.", zone.getName());
if (response.getStatusCode() != HttpStatus.OK) {
log.error("Error while fetching soil moisture from the API. Status code: {}", response.getStatusCode());
log.info("Could not fetch soil moisture data for zone {}.", zone.getName());
throw new BusinessException(ErrorMessage.builder()
.error(Error.AGRANIMO_COULD_NOT_FETCH_SOIL_MOISTURE_FOR_ZONE)
.message("Could not fetch soil moisture.")
.build());
} else {
log.info("Successfully fetched soil moisture from the API.");
log.info("Successfully fetched soil moisture data for zone {}.", zone.getName());
var soilMoistures = response.getBody();
if (null != soilMoistures) {
log.info("Successfully fetched {} soil moisture data points for zone {}.", soilMoistures.length, zone.getName());
return List.of(soilMoistures);
} else {
log.info("Successfully fetched soil moisture from the API.");
log.debug("Response body: {}", response.getBody());
log.info("Successfully fetched soil moisture data for zone {}.", zone.getName());
log.info("Could not fetch soil moisture data for zone {}.", zone.getName());
throw new BusinessException(ErrorMessage.builder()
.error(Error.AGRANIMO_COULD_NOT_FETCH_SOIL_MOISTURE_FOR_ZONE)
.message("Could not fetch soil moisture.")
.build());
}
} catch (Exception e) {
log.error("Error while fetching soil moisture from the API.", e);
log.info("Could not fetch soil moisture data for zone {}.", zone.getName());
}
});
} catch (Exception e) {
log.error("Error while fetching soil moisture from the API.", e);
log.info("Could not fetch soil moisture data for zone {}.", zone.getName());
throw new BusinessException(ErrorMessage.builder()
.error(Error.AGRANIMO_COULD_NOT_FETCH_SOIL_MOISTURE)
.message("Could not fetch soil moisture.")
.build());
}
}

}
6 changes: 3 additions & 3 deletions ...gla/integration/agranimo/ZoneService.java → ...gration/agranimo/AgranimoZoneService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import de.app.fivegla.api.ErrorMessage;
import de.app.fivegla.api.exceptions.BusinessException;
import de.app.fivegla.integration.agranimo.cache.UserDataCache;
import de.app.fivegla.integration.agranimo.dto.Zone;
import de.app.fivegla.integration.agranimo.model.Zone;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
Expand All @@ -22,12 +22,12 @@
@Slf4j
@Service
@RequiredArgsConstructor
public class ZoneService {
public class AgranimoZoneService {

@Value("${app.sensors.agranimo.url}")
private String url;

private final LoginIntegrationService loginService;
private final AgranimoLoginIntegrationService loginService;
private final UserDataCache userDataCache;
private final RestTemplate restTemplate;

Expand Down
Loading

0 comments on commit 692006a

Please sign in to comment.