Merge branch 'main' of https://github.com/vitrum-connect/5gla-api int…

…o feature/add-custom-endpoint-to-end-tx-and-start-orthofoto

# Conflicts:
#	src/main/resources/application.yml

README.md

@@ -8,36 +8,6 @@ BMVI).The website of the project is https://www.5gla.de/, you can find all addit
 interested in the source code,you can find it on
 GitHub: https://github.com/vitrum-connect/5gla-sensor-integration-services
 
-## Run the application within the IDE
-
-To run the application you need to set multiple environment variables. The easiest way to do this is to use the
-following template and replace the values with your own ones:
-
-```
-  CONTEXT_BROKER_URL=https://localhost:1026/;
-  MICROSTREAM_STORAGE_DIRECTORY=/opt/application/.microstream;
-  IMAGE_PATH_BASE_URL=http://localhost:8080/api/images/;
-  API_KEY=___CHANGE_ME___;
-  NOTIFICATION_URLS=https://localhost:5050/notify;
-  SPRING_PROFILES_ACTIVE=maintenance;
-  CORS_ALLOWED_ORIGINS=http://localhost:8080;
-  CONTEXT_PATH=/api;
-```
-
-The following table describes the environment variables:
-
-| Environment Variable          | Description                                             |
-|-------------------------------|---------------------------------------------------------|
-| CONTEXT_BROKER_URL            | The URL of the Orion Context Broker.                    |
-| MICROSTREAM_STORAGE_DIRECTORY | The directory where the Microstream database is stored. |
-| IMAGE_PATH_BASE_URL           | The base URL of the image path.                         |
-| SENTEK_API_TOKEN              | The API token of the Sentek account.                    |
-| API_KEY                       | The API key of the application.                         |
-| NOTIFICATION_URLS             | The URL of the notification service.                    |
-| SPRING_PROFILES_ACTIVE        | The active Spring profile.                              |
-| CORS_ALLOWED_ORIGINS          | The allowed origins for CORS.                           |
-| CONTEXT_PATH                  | The context path of the application.                    |
-
 ## Build the project
 
 To build the project and to resolve the dependencies from GitHub Packages you need to create a personal access token and

src/main/java/de/app/fivegla/business/ThirdPartyApiConfigurationService.java

@@ -1,13 +1,17 @@
 package de.app.fivegla.business;
 
+import de.app.fivegla.event.events.HistoricalDataImportEvent;
 import de.app.fivegla.persistence.ThirdPartyApiConfigurationRepository;
 import de.app.fivegla.persistence.entity.ThirdPartyApiConfiguration;
 import jakarta.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.stereotype.Service;
 
 import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
 import java.util.Date;
 import java.util.List;
 import java.util.Optional;
@@ -18,6 +22,7 @@
 public class ThirdPartyApiConfigurationService {
 
     private final ThirdPartyApiConfigurationRepository thirdPartyApiConfigurationRepository;
+    private final ApplicationEventPublisher applicationEventPublisher;
 
     /**
      * Creates a third-party API configuration and adds it to the system.
@@ -85,4 +90,18 @@ public void updateLastRun(ThirdPartyApiConfiguration thirdPartyApiConfiguration)
     public Optional<ThirdPartyApiConfiguration> findById(Long id) {
         return thirdPartyApiConfigurationRepository.findById(id);
     }
+
+    /**
+     * Triggers the import of historical data for a given tenant and UUID starting from a specific date.
+     *
+     * @param tenantId           the ID of the tenant for which the import will be triggered
+     * @param uuid               the UUID associated with the tenant
+     * @param startDateInThePast the start date from which the import will begin, in the past
+     */
+    public void triggerImport(String tenantId, String uuid, LocalDate startDateInThePast) {
+        getThirdPartyApiConfigurations(tenantId, uuid).forEach(thirdPartyApiConfiguration -> {
+            applicationEventPublisher.publishEvent(new HistoricalDataImportEvent(thirdPartyApiConfiguration.getId(),
+                    startDateInThePast.atStartOfDay(ZoneId.systemDefault()).toInstant()));
+        });
+    }
 }

src/main/java/de/app/fivegla/controller/tenant/ThirdPartyApiConfigurationController.java

@@ -18,11 +18,13 @@
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.format.annotation.DateTimeFormat;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 
 import java.security.Principal;
+import java.time.LocalDate;
 
 /**
  * The JobController class handles requests related to job operations.
@@ -209,4 +211,32 @@ public ResponseEntity<? extends Response> deleteThirdPartyApiConfiguration(@Path
         return ResponseEntity.ok().body(new Response());
     }
 
+    @Operation(
+            summary = "Triggers a third-party API configuration to fulfill an import in the past.",
+            description = "Triggers a third-party API configuration to fulfill an import in the past.",
+            tags = BaseMappings.THIRD_PARTY_API_CONFIGURATION
+    )
+    @ApiResponse(
+            responseCode = "200",
+            description = "The third party API configuration was triggered successfully.",
+            content = @Content(
+                    mediaType = MediaType.APPLICATION_JSON_VALUE,
+                    schema = @Schema(implementation = Response.class)
+            )
+    )
+    @ApiResponse(
+            responseCode = "400",
+            description = "The request is invalid.",
+            content = @Content(
+                    mediaType = MediaType.APPLICATION_JSON_VALUE,
+                    schema = @Schema(implementation = Response.class)
+            )
+    )
+    @PostMapping(value = "/{uuid}/trigger/{startDateInThePast}", produces = MediaType.APPLICATION_JSON_VALUE)
+    public ResponseEntity<? extends Response> triggerThirdPartyApiConfigurationInThePast(@PathVariable(value = "uuid") String uuid, @PathVariable(value = "startDateInThePast") @DateTimeFormat(pattern = "yyyy-MM-dd") LocalDate startDateInThePast, Principal principal) {
+        var tenant = validateTenant(tenantService, principal);
+        thirdPartyApiConfigurationService.triggerImport(tenant.getTenantId(), uuid, startDateInThePast);
+        return ResponseEntity.ok().body(new Response());
+    }
+
 }

src/main/java/de/app/fivegla/event/DataImportEventHandler.java

@@ -6,6 +6,7 @@
 import de.app.fivegla.business.TenantService;
 import de.app.fivegla.business.ThirdPartyApiConfigurationService;
 import de.app.fivegla.event.events.DataImportEvent;
+import de.app.fivegla.event.events.HistoricalDataImportEvent;
 import de.app.fivegla.integration.agranimo.AgranimoMeasurementImport;
 import de.app.fivegla.integration.agvolution.AgvolutionMeasurementImport;
 import de.app.fivegla.integration.farm21.Farm21MeasurementImport;
@@ -65,4 +66,33 @@ public void handleDataImportEvent(DataImportEvent dataImportEvent) {
         thirdPartyApiConfigurationService.updateLastRun(thirdPartyApiConfiguration);
     }
 
+    @EventListener(HistoricalDataImportEvent.class)
+    public void handleHistoricalDataImportEvent(HistoricalDataImportEvent historicalDataImportEvent) {
+        var thirdPartyApiConfiguration = thirdPartyApiConfigurationService.findById(historicalDataImportEvent.thirdPartyApiConfigurationId())
+                .orElseThrow(() -> new BusinessException(ErrorMessage.builder()
+                        .error(Error.THIRD_PARTY_API_CONFIGURATION_NOT_FOUND)
+                        .message("Third party API configuration not found.")
+                        .build()));
+        log.info("Handling data import event for tenant {} and manufacturer {}.", thirdPartyApiConfiguration.getTenant().getTenantId(), thirdPartyApiConfiguration.getManufacturer());
+        var manufacturer = thirdPartyApiConfiguration.getManufacturer();
+        var tenantId = thirdPartyApiConfiguration.getTenant().getTenantId();
+        var optionalTenant = tenantService.findByTenantId(tenantId);
+        if (optionalTenant.isEmpty()) {
+            log.error("Tenant with id {} not found, not able to handle data import event", tenantId);
+        } else {
+            var tenant = optionalTenant.get();
+            switch (manufacturer) {
+                case SOILSCOUT -> soilScoutScheduledMeasurementImport.run(tenant, thirdPartyApiConfiguration, historicalDataImportEvent.startDate());
+                case AGVOLUTION -> agvolutionMeasurementImport.run(tenant, thirdPartyApiConfiguration, historicalDataImportEvent.startDate());
+                case AGRANIMO -> agranimoMeasurementImport.run(tenant, thirdPartyApiConfiguration, historicalDataImportEvent.startDate());
+                case FARM21 -> farm21MeasurementImport.run(tenant, thirdPartyApiConfiguration, historicalDataImportEvent.startDate());
+                case SENSOTERRA -> sensoterraMeasurementImport.run(tenant, thirdPartyApiConfiguration, historicalDataImportEvent.startDate());
+                case SENTEK -> sentekMeasurementImport.run(tenant, thirdPartyApiConfiguration, historicalDataImportEvent.startDate());
+                case WEENAT -> weenatMeasurementImport.run(tenant, thirdPartyApiConfiguration, historicalDataImportEvent.startDate());
+                default -> throw new IllegalArgumentException("Unknown manufacturer: " + manufacturer);
+            }
+        }
+        thirdPartyApiConfigurationService.updateLastRun(thirdPartyApiConfiguration);
+    }
+
 }

src/main/java/de/app/fivegla/event/events/HistoricalDataImportEvent.java

@@ -0,0 +1,11 @@
+package de.app.fivegla.event.events;
+
+import java.time.Instant;
+
+/**
+ * Event for data import.
+ *
+ * @param thirdPartyApiConfigurationId The ID of the third party API configuration.
+ */
+public record HistoricalDataImportEvent(Long thirdPartyApiConfigurationId, Instant startDate) {
+}

src/main/java/de/app/fivegla/integration/agranimo/AgranimoMeasurementImport.java

@@ -69,6 +69,36 @@ public void run(Tenant tenant, ThirdPartyApiConfiguration thirdPartyApiConfigura
         }
     }
 
+    /**
+     * Asynchronously runs the scheduled data import from Agranimo API.
+     *
+     * @param tenant                     The tenant to import data for.
+     * @param thirdPartyApiConfiguration The configuration for the third-party API.
+     * @param start                      The start time for fetching water content data.
+     */
+    @Async
+    public void run(Tenant tenant, ThirdPartyApiConfiguration thirdPartyApiConfiguration, Instant start) {
+        var begin = Instant.now();
+        try {
+            agranimoZoneService.fetchZones(thirdPartyApiConfiguration).forEach(zone -> {
+                var waterContent = agranimoSoilMoistureIntegrationService.fetchWaterContent(thirdPartyApiConfiguration, zone, start);
+                jobMonitor.logNrOfEntitiesFetched(Manufacturer.AGRANIMO, waterContent.size());
+                log.info("Found {} water content entries", waterContent.size());
+                log.info("Persisting {} water content entries", waterContent.size());
+                waterContent.forEach(
+                        soilMoisture -> persistDataWithinFiware(tenant, zone, soilMoisture)
+                );
+            });
+        } catch (Exception e) {
+            log.error("Error while running scheduled data import from Agranimo API", e);
+            jobMonitor.logErrorDuringExecution(Manufacturer.AGRANIMO);
+        } finally {
+            log.info("Finished scheduled data import from Agranimo API");
+            var end = Instant.now();
+            jobMonitor.logJobExecutionTime(Manufacturer.AGRANIMO, begin.until(end, ChronoUnit.SECONDS));
+        }
+    }
+
     private void persistDataWithinFiware(Tenant tenant, Zone zone, SoilMoisture soilMoisture) {
         try {
             fiwareIntegrationServiceWrapper.persist(tenant, zone, soilMoisture);

src/main/java/de/app/fivegla/integration/agvolution/AgvolutionMeasurementImport.java

@@ -61,6 +61,29 @@ public void run(Tenant tenant, ThirdPartyApiConfiguration thirdPartyApiConfigura
         }
     }
 
+    /**
+     * Run historical data import.
+     */
+    @Async
+    public void run(Tenant tenant, ThirdPartyApiConfiguration thirdPartyApiConfiguration, Instant start) {
+        var begin = Instant.now();
+        try {
+            log.info("Running historical data import from Agvolution API, this may take a while");
+            var seriesEntries = agvolutionSensorDataIntegrationService.fetchAll(thirdPartyApiConfiguration, start);
+            log.info("Found {} seriesEntries", seriesEntries.size());
+            log.info("Persisting {} seriesEntries", seriesEntries.size());
+            jobMonitor.logNrOfEntitiesFetched(Manufacturer.AGVOLUTION, seriesEntries.size());
+            seriesEntries.forEach(seriesEntry -> persistDataWithinFiware(tenant, seriesEntry));
+        } catch (Exception e) {
+            log.error("Error while running scheduled data import from Agvolution API", e);
+            jobMonitor.logErrorDuringExecution(Manufacturer.AGVOLUTION);
+        } finally {
+            log.info("Finished scheduled data import from Agvolution API");
+            var end = Instant.now();
+            jobMonitor.logJobExecutionTime(Manufacturer.AGVOLUTION, begin.until(end, ChronoUnit.SECONDS));
+        }
+    }
+
     private void persistDataWithinFiware(Tenant tenant, SeriesEntry seriesEntry) {
         try {
             agvolutionFiwareIntegrationServiceWrapper.persist(tenant, seriesEntry);

src/main/java/de/app/fivegla/integration/farm21/Farm21MeasurementImport.java

@@ -64,6 +64,29 @@ public void run(Tenant tenant, ThirdPartyApiConfiguration thirdPartyApiConfigura
         }
     }
 
+    /**
+     * Run historical data import.
+     */
+    @Async
+    public void run(Tenant tenant, ThirdPartyApiConfiguration thirdPartyApiConfiguration, Instant start) {
+        var begin = Instant.now();
+        try {
+            log.info("Running historical data import from Farm21 API, this may take a while");
+            var measurements = farm21SensorDataIntegrationService.fetchAll(thirdPartyApiConfiguration, start, Instant.now());
+            log.info("Found {} measurements", measurements.size());
+            log.info("Persisting {} measurements", measurements.size());
+            jobMonitor.logNrOfEntitiesFetched(Manufacturer.FARM21, measurements.size());
+            measurements.entrySet().forEach(sensorListEntry -> persistDataWithinFiware(tenant, sensorListEntry));
+        } catch (Exception e) {
+            log.error("Error while running scheduled data import from Farm21 API", e);
+            jobMonitor.logErrorDuringExecution(Manufacturer.FARM21);
+        } finally {
+            log.info("Finished scheduled data import from Farm21 API");
+            var end = Instant.now();
+            jobMonitor.logJobExecutionTime(Manufacturer.FARM21, begin.until(end, ChronoUnit.SECONDS));
+        }
+    }
+
     private void persistDataWithinFiware(Tenant tenant, Map.Entry<Sensor, List<SensorData>> entry) {
         try {
             Sensor key = entry.getKey();

src/main/java/de/app/fivegla/integration/fiware/model/MicaSenseImage.java

@@ -0,0 +1,67 @@
+package de.app.fivegla.integration.fiware.model;
+
+import de.app.fivegla.integration.fiware.model.api.FiwareEntity;
+import de.app.fivegla.integration.fiware.model.api.Validatable;
+import de.app.fivegla.integration.fiware.model.internal.Attribute;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Device measurement.
+ */
+@Slf4j
+public record MicaSenseImage(
+        String id,
+        String type,
+        Attribute group,
+        Attribute oid,
+        Attribute droneId,
+        Attribute transactionId,
+        Attribute imageChannel,
+        Attribute base64encodedImage,
+        Attribute imagePath,
+        Attribute dateCreated,
+        double latitude,
+        double longitude
+) implements FiwareEntity, Validatable {
+
+    @Override
+    public String asJson() {
+        validate();
+        var json = "{" +
+                "  \"id\":\"" + id.trim() + "\"," +
+                "  \"type\":\"" + type.trim() + "\"," +
+                "  \"customGroup\":" + group.asJson().trim() + "," +
+                "  \"oid\":" + oid.asJson().trim() + "," +
+                "  \"droneId\":" + droneId.asJson().trim() + "," +
+                "  \"transactionId\":" + transactionId.asJson().trim() + "," +
+                "  \"imageChannel\":" + imageChannel.asJson().trim() + "," +
+                "  \"base64encodedImage\":" + base64encodedImage.asJson().trim() + "," +
+                "  \"imagePath\":" + imagePath.asJson().trim() + "," +
+                "  \"dateCreated\":" + dateCreated.asJson().trim() + "," +
+                "  \"location\":" + locationAsJson(latitude, longitude).trim() +
+                "}";
+        log.debug("{} as JSON: {}", this.getClass().getSimpleName(), json);
+        return json;
+    }
+
+    @Override
+    public void validate() {
+        if (StringUtils.isBlank(id)) {
+            throw new IllegalArgumentException("The id of the device measurement must not be null or blank.");
+        }
+        if (StringUtils.isBlank(type)) {
+            throw new IllegalArgumentException("The type of the device measurement must not be null or blank.");
+        }
+    }
+
+    @Override
+    public String getId() {
+        return id;
+    }
+
+    @Override
+    public String getType() {
+        return type;
+    }
+}

src/main/java/de/app/fivegla/integration/imageprocessing/ImageProcessingFiwareIntegrationServiceWrapper.java

@@ -3,9 +3,7 @@
 
 import de.app.fivegla.api.enums.EntityType;
 import de.app.fivegla.integration.fiware.FiwareEntityIntegrationService;
-import de.app.fivegla.integration.fiware.model.DeviceMeasurement;
-import de.app.fivegla.integration.fiware.model.internal.DateAttribute;
-import de.app.fivegla.integration.fiware.model.internal.EmptyAttribute;
+import de.app.fivegla.integration.fiware.model.MicaSenseImage;
 import de.app.fivegla.integration.fiware.model.internal.TextAttribute;
 import de.app.fivegla.persistence.entity.Group;
 import de.app.fivegla.persistence.entity.Image;
@@ -33,14 +31,17 @@ public class ImageProcessingFiwareIntegrationServiceWrapper {
      * @param image the image to create the measurement for
      */
     public void createDroneDeviceMeasurement(Tenant tenant, Group group, String droneId, Image image) {
-        var deviceMeasurement = new DeviceMeasurement(
+        var deviceMeasurement = new MicaSenseImage(
                 tenant.getFiwarePrefix() + droneId,
                 EntityType.MICASENSE_IMAGE.getKey(),
                 new TextAttribute(group.getOid()),
-                new TextAttribute("image"),
-                new EmptyAttribute(),
-                new DateAttribute(image.getMeasuredAt()),
-                new TextAttribute(imagePathBaseUrl + image.getOid()),
+                new TextAttribute(image.getOid()),
+                new TextAttribute(droneId),
+                new TextAttribute(image.getTransactionId()),
+                new TextAttribute(image.getChannel().name()),
+                new TextAttribute(image.getBase64encodedImage()),
+                new TextAttribute(imagePathBaseUrl + image.getFullFilename(tenant)),
+                new TextAttribute(image.getMeasuredAt().toString()),
                 image.getLatitude(),
                 image.getLongitude());
         fiwareEntityIntegrationService.persist(tenant, group, deviceMeasurement);