diff --git a/pom.xml b/pom.xml index 4aafb71..9ad72e1 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ com.oracle.database.jdbc ojdbc8 - 21.3.0.0 + 21.9.0.0 diff --git a/src/main/java/fr/abes/logskbart/entity/bacon/LigneKbart.java b/src/main/java/fr/abes/logskbart/entity/bacon/LigneKbart.java index 9f2922c..a0567cc 100644 --- a/src/main/java/fr/abes/logskbart/entity/bacon/LigneKbart.java +++ b/src/main/java/fr/abes/logskbart/entity/bacon/LigneKbart.java @@ -8,7 +8,7 @@ import java.util.Date; @Entity -@Table(name = "LIGNE_KBART_CONVERGENCE") +@Table(name = "LIGNE_KBART") @Setter @Getter public class LigneKbart implements Serializable { @Id @@ -65,13 +65,15 @@ public class LigneKbart implements Serializable { private String precedeingPublicationTitleId; @Column(name = "ACCESS_TYPE") private String accessType; - @ManyToOne(targetEntity = ProviderPackage.class, cascade = CascadeType.REMOVE, optional = false) - @JoinColumns({ - @JoinColumn(name = "PROVIDER_PACKAGE_PACKAGE", referencedColumnName = "PACKAGE"), - @JoinColumn(name = "PROVIDER_PACKAGE_DATE_P", referencedColumnName = "DATE_P"), - @JoinColumn(name = "PROVIDER_PACKAGE_IDT_PROVIDER", referencedColumnName = "PROVIDER_IDT_PROVIDER" - )}) - private ProviderPackage providerPackage; + @Column(name = "PACKAGE") + private String packageName; + @Column(name = "DATE_P") + private Date dateP; + @Column(name = "PROVIDER_IDT_PROVIDER") + private Integer providerIdtProvider; @Column(name = "BEST_PPN") private String bestPpn; + @ManyToOne(targetEntity = ProviderPackage.class, cascade = CascadeType.REMOVE, optional = false) + @JoinColumn(name = "ID_PROVIDER_PACKAGE", referencedColumnName = "ID_PROVIDER_PACKAGE") + private ProviderPackage providerPackage; } diff --git a/src/main/java/fr/abes/logskbart/entity/bacon/ProviderPackage.java b/src/main/java/fr/abes/logskbart/entity/bacon/ProviderPackage.java index 0f4421b..e1c62b5 100644 --- a/src/main/java/fr/abes/logskbart/entity/bacon/ProviderPackage.java +++ b/src/main/java/fr/abes/logskbart/entity/bacon/ProviderPackage.java @@ -7,6 +7,7 @@ import lombok.Setter; import java.io.Serializable; +import java.util.Date; @Entity @Table(name = "PROVIDER_PACKAGE") @@ -14,8 +15,16 @@ @AllArgsConstructor @NoArgsConstructor public class ProviderPackage implements Serializable { - @EmbeddedId - private ProviderPackageId providerPackageId; + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "ID_PROVIDER_PACKAGE") + private Integer providerPackageId; + @Column(name = "PACKAGE") + private String packageName; + @Column(name = "DATE_P") + private Date dateP; + @Column(name = "PROVIDER_IDT_PROVIDER") + private Integer providerIdtProvider; @Column(name = "LABEL_ABES") private char labelAbes; @@ -24,8 +33,10 @@ public class ProviderPackage implements Serializable { @JoinColumn(referencedColumnName = "IDT_PROVIDER", insertable = false, updatable = false) private Provider provider; - public ProviderPackage(ProviderPackageId providerPackageId, char labelAbes) { - this.providerPackageId = providerPackageId; + public ProviderPackage(String packageName, Date dateP, Integer providerIdtProvider, char labelAbes) { + this.packageName = packageName; + this.dateP = dateP; + this.providerIdtProvider = providerIdtProvider; this.labelAbes = labelAbes; } } diff --git a/src/main/java/fr/abes/logskbart/entity/bacon/ProviderPackageId.java b/src/main/java/fr/abes/logskbart/entity/bacon/ProviderPackageId.java deleted file mode 100644 index 0c7d45f..0000000 --- a/src/main/java/fr/abes/logskbart/entity/bacon/ProviderPackageId.java +++ /dev/null @@ -1,46 +0,0 @@ -package fr.abes.logskbart.entity.bacon; - -import jakarta.persistence.Column; -import jakarta.persistence.Embeddable; -import jakarta.persistence.JoinColumn; -import jakarta.persistence.ManyToOne; -import lombok.Getter; -import lombok.NoArgsConstructor; - -import java.io.Serializable; -import java.util.Date; -import java.util.Objects; - - -@Embeddable -@Getter -@NoArgsConstructor -public class ProviderPackageId implements Serializable { - @Column(name = "PACKAGE") - private String packageName; - @Column(name = "DATE_P") - private Date dateP; - @Column(name = "PROVIDER_IDT_PROVIDER") - private Integer providerIdtProvider; - - public ProviderPackageId(String packageName, Date datePackage, Integer idtProvider) { - this.packageName = packageName; - this.dateP = datePackage; - this.providerIdtProvider = idtProvider; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof ProviderPackageId)) return false; - ProviderPackageId that = (ProviderPackageId) o; - return Objects.equals(getProviderIdtProvider(), that.getProviderIdtProvider()) && - Objects.equals(getDateP(), that.getDateP()) && - Objects.equals(getPackageName(), that.getPackageName()); - } - - @Override - public int hashCode() { - return Objects.hash(getProviderIdtProvider(), getDateP(), getPackageName()); - } -} diff --git a/src/main/java/fr/abes/logskbart/kafka/KbartListener.java b/src/main/java/fr/abes/logskbart/kafka/KbartListener.java deleted file mode 100644 index c5f9bda..0000000 --- a/src/main/java/fr/abes/logskbart/kafka/KbartListener.java +++ /dev/null @@ -1,48 +0,0 @@ -package fr.abes.logskbart.kafka; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import fr.abes.logskbart.dto.PackageKbartDto; -import fr.abes.logskbart.dto.PackageKbartDtoKafka; -import fr.abes.logskbart.exception.IllegalDateException; -import fr.abes.logskbart.exception.IllegalPackageException; -import fr.abes.logskbart.exception.IllegalProviderException; -import fr.abes.logskbart.service.KbartService; -import fr.abes.logskbart.utils.Utils; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.stereotype.Service; - - -@Service -@Slf4j -public class KbartListener { - @Autowired - private ObjectMapper mapper; - - @Autowired - private KbartService service; - - @KafkaListener(topics = {"packageKbart"}, groupId = "lignesKbart", containerFactory = "kafkaKbartListenerContainerFactory") - public void listenKbartFromKafka(ConsumerRecord lignesKbart) { - PackageKbartDto packageKbartDto = new PackageKbartDto(); - try { - packageKbartDto.setPackageName(Utils.extractPackageName(lignesKbart.key())); - packageKbartDto.setProvider(Utils.extractProvider(lignesKbart.key())); - packageKbartDto.setDatePackage(Utils.extractDate(lignesKbart.key())); - PackageKbartDtoKafka packageFromKafka = mapper.readValue(lignesKbart.value(), PackageKbartDtoKafka.class); - packageKbartDto.setLigneKbartDtos(packageFromKafka.getKbartDtos()); - - //traitement de la liste de dto à traiter en fonction - service.chargerKbart(packageKbartDto); - } catch (IllegalPackageException | IllegalProviderException | IllegalDateException e) { - log.error("Erreur dans les données en entrée, provider / nom de package ou format de date incorrect"); - } catch (JsonProcessingException e) { - log.error(e.getMessage()); - } - } - - -} diff --git a/src/main/java/fr/abes/logskbart/kafka/LogsListener.java b/src/main/java/fr/abes/logskbart/kafka/LogsListener.java index f6f2726..44d3a1a 100644 --- a/src/main/java/fr/abes/logskbart/kafka/LogsListener.java +++ b/src/main/java/fr/abes/logskbart/kafka/LogsListener.java @@ -1,19 +1,27 @@ package fr.abes.logskbart.kafka; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import fr.abes.logskbart.dto.Kbart2KafkaDto; import fr.abes.logskbart.entity.logs.LogKbart; import fr.abes.logskbart.repository.logs.LogKbartRepository; import fr.abes.logskbart.utils.UtilsMapper; +import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.header.Header; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.stereotype.Service; +import java.io.File; +import java.io.IOException; +import java.nio.file.*; import java.sql.Date; import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +@Slf4j @Service public class LogsListener { @@ -26,13 +34,80 @@ public class LogsListener { @Autowired private LogKbartRepository repository; - @KafkaListener(topics = {"infokbart2kafka", "errorkbart2kafka"}, groupId = "logskbart", containerFactory = "kafkaLogsListenerContainerFactory") - public void listenInfoKbart2KafkaAndErrorKbart2Kafka(ConsumerRecord message) throws JsonProcessingException { - Kbart2KafkaDto dto = mapper.readValue(message.value(), Kbart2KafkaDto.class); - LogKbart entity = logsMapper.map(dto, LogKbart.class); - Timestamp timestamp = new Timestamp(message.timestamp()); - entity.setTimestamp(new Date(timestamp.getTime())); - entity.setPackageName(message.key()); - repository.save(entity); + /** + * Ecoute les topic de log d'erreurs et de fin de traitement bestPpn et génère un fichier err pour chaque fichier kbart + * @param message le message kafka + * @throws IOException exception levée + */ + @KafkaListener(topics = {"errorkbart2kafka", "bestppn.endoftraitment"}, groupId = "logskbart", containerFactory = "kafkaLogsListenerContainerFactory") + public void listenInfoKbart2KafkaAndErrorKbart2Kafka(ConsumerRecord message) throws IOException { + + if (message.topic().equals("errorkbart2kafka")) { + Kbart2KafkaDto dto = mapper.readValue(message.value(), Kbart2KafkaDto.class); + LogKbart entity = logsMapper.map(dto, LogKbart.class); + Timestamp timestamp = new Timestamp(message.timestamp()); + entity.setTimestamp(new Date(timestamp.getTime())); + entity.setPackageName(message.key().replaceAll("\\[line\\s:\\s\\d+\\]", "")); + + // Si la ligne de log sur le topic errorkbart2kafka est de type ERROR + if (entity.getLevel().toString().equals("ERROR")) { + String nbrLine = message.key().substring(message.key().indexOf(".tsv")+4).replaceAll("\\[line\\s:\\s", "").replaceAll("]", ""); + String fileName = message.key().replaceAll(".tsv\\[line\\s:\\s\\d+\\]", ".err"); + String line = nbrLine + "\t" + dto.getMessage(); + + // Vérifie qu'un fichier portant le nom du kbart en cours existe + Path of = Path.of(fileName); + if (Files.exists(of)) { + // Inscrit la ligne dedans + Files.write(of, (line + System.lineSeparator()).getBytes(), StandardOpenOption.APPEND); + } else if (!Files.exists(of)) { + try { + // Créer le fichier et inscrit la ligne dedans + Files.createFile(of); + // Créer la ligne d'en-tête + Files.write(of, ("LINE\tMESSAGE" + System.lineSeparator()).getBytes(), StandardOpenOption.APPEND); + // Inscrit les informations sur la ligne + Files.write(of, (line + System.lineSeparator()).getBytes(), StandardOpenOption.APPEND); + log.info("Fichier temporaire créé."); + } catch (SecurityException | IOException e) { + log.error("Erreur lors de la création du fichier temporaire. " + e); + throw new RuntimeException(e); + } + } + } + + // Inscrit l'entity en BDD + repository.save(entity); + + } else { // Si la ligne sur le topic bestppn.endoftraitment contient OK + + // Créer un nouveau Path avec le FileName (en remplaçant l'extension par .err) + Path source = null; + for (Header header : message.headers().toArray()) { + if (header.key().equals("FileName")) { + source = Path.of(new String(header.value()).replaceAll(".tsv", ".err")); + break; + } + } + + // Copie le fichier existant vers le répertoire temporaire en ajoutant sa date de création + if (source != null && Files.exists(source)) { + LocalDateTime time = LocalDateTime.now(); + DateTimeFormatter format = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss", Locale.FRANCE); + String date = format.format(time); + + // Vérification du chemin et création si inexistant + String tempLog = "tempLog/"; + File chemin = new File("tempLog/"); + if (!chemin.isDirectory()) { + Files.createDirectory(Paths.get(tempLog)); + } + Path target = Path.of("tempLog\\" + date + "_" + source); + + // Déplacement du fichier + Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); + log.info("Fichier de log transféré dans le dossier temporaire."); + } + } } } diff --git a/src/main/java/fr/abes/logskbart/repository/bacon/ProviderPackageRepository.java b/src/main/java/fr/abes/logskbart/repository/bacon/ProviderPackageRepository.java index 7a6cc33..f0cbc94 100644 --- a/src/main/java/fr/abes/logskbart/repository/bacon/ProviderPackageRepository.java +++ b/src/main/java/fr/abes/logskbart/repository/bacon/ProviderPackageRepository.java @@ -2,7 +2,6 @@ import fr.abes.logskbart.configuration.BaconDbConfiguration; import fr.abes.logskbart.entity.bacon.ProviderPackage; -import fr.abes.logskbart.entity.bacon.ProviderPackageId; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; @@ -11,6 +10,6 @@ @Repository @BaconDbConfiguration -public interface ProviderPackageRepository extends JpaRepository { - Optional findByProviderPackageId(ProviderPackageId providerPackageId); +public interface ProviderPackageRepository extends JpaRepository { + Optional findByPackageNameAndDatePAndProviderIdtProvider(String packageName, Date dateP, Integer providerIdtProvider); } diff --git a/src/main/java/fr/abes/logskbart/service/KbartLoader.java b/src/main/java/fr/abes/logskbart/service/KbartLoader.java deleted file mode 100644 index a1a569d..0000000 --- a/src/main/java/fr/abes/logskbart/service/KbartLoader.java +++ /dev/null @@ -1,7 +0,0 @@ -package fr.abes.logskbart.service; - -import fr.abes.logskbart.dto.PackageKbartDto; - -public interface KbartLoader { - void chargerPackageKbart(PackageKbartDto packageKbartDto); -} diff --git a/src/main/java/fr/abes/logskbart/service/KbartLoaderFactory.java b/src/main/java/fr/abes/logskbart/service/KbartLoaderFactory.java deleted file mode 100644 index fb33d99..0000000 --- a/src/main/java/fr/abes/logskbart/service/KbartLoaderFactory.java +++ /dev/null @@ -1,22 +0,0 @@ -package fr.abes.logskbart.service; - -import fr.abes.logskbart.utils.LoaderType; -import org.springframework.stereotype.Service; - -@Service -public class KbartLoaderFactory { - private final KbartToTsvService kbartToTsvService; - private final KbartToBddService kbartToBddService; - - public KbartLoaderFactory(KbartToTsvService kbartToTsvService, KbartToBddService kbartToBddService) { - this.kbartToTsvService = kbartToTsvService; - this.kbartToBddService = kbartToBddService; - } - - public KbartLoader getLoader(LoaderType type) { - return switch (type) { - case tsv -> kbartToTsvService; - case bdd -> kbartToBddService; - }; - } -} diff --git a/src/main/java/fr/abes/logskbart/service/KbartService.java b/src/main/java/fr/abes/logskbart/service/KbartService.java index 1b9aa56..e69de29 100644 --- a/src/main/java/fr/abes/logskbart/service/KbartService.java +++ b/src/main/java/fr/abes/logskbart/service/KbartService.java @@ -1,23 +0,0 @@ -package fr.abes.logskbart.service; - -import fr.abes.logskbart.dto.PackageKbartDto; -import fr.abes.logskbart.utils.LoaderType; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Service; - -@Service -public class KbartService { - private final KbartLoaderFactory factory; - - private final Environment env; - - public KbartService(KbartLoaderFactory factory, Environment env) { - this.factory = factory; - this.env = env; - } - - public void chargerKbart(PackageKbartDto packageKbart) { - KbartLoader loader = factory.getLoader(LoaderType.valueOf(env.getProperty("LOADERTYPE"))); - loader.chargerPackageKbart(packageKbart); - } -} diff --git a/src/main/java/fr/abes/logskbart/service/KbartToBddService.java b/src/main/java/fr/abes/logskbart/service/KbartToBddService.java index 2e84428..e69de29 100644 --- a/src/main/java/fr/abes/logskbart/service/KbartToBddService.java +++ b/src/main/java/fr/abes/logskbart/service/KbartToBddService.java @@ -1,58 +0,0 @@ -package fr.abes.logskbart.service; - -import fr.abes.logskbart.dto.PackageKbartDto; -import fr.abes.logskbart.entity.bacon.LigneKbart; -import fr.abes.logskbart.entity.bacon.Provider; -import fr.abes.logskbart.entity.bacon.ProviderPackage; -import fr.abes.logskbart.entity.bacon.ProviderPackageId; -import fr.abes.logskbart.repository.bacon.LigneKbartRepository; -import fr.abes.logskbart.repository.bacon.ProviderPackageRepository; -import fr.abes.logskbart.repository.bacon.ProviderRepository; -import fr.abes.logskbart.utils.UtilsMapper; -import org.springframework.stereotype.Service; - -import java.util.List; -import java.util.Optional; - -@Service -public class KbartToBddService implements KbartLoader { - private final ProviderPackageRepository providerPackageRepository; - private final ProviderRepository providerRepository; - private final LigneKbartRepository ligneKbartRepository; - - private final UtilsMapper mapper; - - public KbartToBddService(ProviderPackageRepository providerPackageRepository, ProviderRepository providerRepository, LigneKbartRepository ligneKbartRepository, UtilsMapper mapper) { - this.providerPackageRepository = providerPackageRepository; - this.providerRepository = providerRepository; - this.ligneKbartRepository = ligneKbartRepository; - this.mapper = mapper; - } - - @Override - public void chargerPackageKbart(PackageKbartDto packageKbartDto) { - ProviderPackage providerPackage = handlePackageAndProvider(packageKbartDto); - List ligneKbarts = mapper.mapList(packageKbartDto.getLigneKbartDtos(), LigneKbart.class); - ligneKbarts.forEach(ligne -> ligne.setProviderPackage(providerPackage)); - ligneKbartRepository.saveAll(ligneKbarts); - } - - private ProviderPackage handlePackageAndProvider(PackageKbartDto packageKbartDto) { - //on vérifie si le package existe dans la base pour une date de chargement donnée - //récupération provider - Optional providerOpt = providerRepository.findByProvider(packageKbartDto.getProvider()); - if (providerOpt.isPresent()) { - Provider provider = providerOpt.get(); - ProviderPackageId providerPackageId = new ProviderPackageId(packageKbartDto.getPackageName(), packageKbartDto.getDatePackage(), provider.getIdtProvider()); - Optional providerPackage = providerPackageRepository.findByProviderPackageId(providerPackageId); - //pas d'info de package, on le crée - return providerPackage.orElseGet(() -> providerPackageRepository.save(new ProviderPackage(providerPackageId, 'N'))); - } else { - //pas de provider, ni de package, on les crée tous les deux - Provider newProvider = new Provider(packageKbartDto.getProvider()); - Provider savedProvider = providerRepository.save(newProvider); - ProviderPackage providerPackage = new ProviderPackage(new ProviderPackageId(packageKbartDto.getPackageName(), packageKbartDto.getDatePackage(), savedProvider.getIdtProvider()), 'N'); - return providerPackageRepository.save(providerPackage); - } - } -} diff --git a/src/main/java/fr/abes/logskbart/service/KbartToTsvService.java b/src/main/java/fr/abes/logskbart/service/KbartToTsvService.java index 0ea2801..e69de29 100644 --- a/src/main/java/fr/abes/logskbart/service/KbartToTsvService.java +++ b/src/main/java/fr/abes/logskbart/service/KbartToTsvService.java @@ -1,47 +0,0 @@ -package fr.abes.logskbart.service; - -import com.opencsv.bean.CsvBindByName; -import com.opencsv.bean.CsvBindByPosition; -import com.opencsv.bean.StatefulBeanToCsv; -import com.opencsv.bean.StatefulBeanToCsvBuilder; -import com.opencsv.exceptions.CsvDataTypeMismatchException; -import com.opencsv.exceptions.CsvRequiredFieldEmptyException; -import fr.abes.logskbart.dto.LigneKbartDto; -import fr.abes.logskbart.dto.PackageKbartDto; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import java.io.FileWriter; -import java.io.IOException; -import java.io.Writer; -import java.util.Arrays; -import java.util.Comparator; -import java.util.stream.Collectors; - -@Service -public class KbartToTsvService implements KbartLoader { - @Value("${path.tsvFile}") - private String pathToFichier; - - @Override - public void chargerPackageKbart(PackageKbartDto packageKbartDto) { - try (Writer writer = new FileWriter(pathToFichier + packageKbartDto.getPackageName())) { - writer.append(buildHeader(LigneKbartDto.class)); - StatefulBeanToCsv sbc = new StatefulBeanToCsvBuilder(writer).withSeparator('\t').withApplyQuotesToAll(false).build(); - sbc.write(packageKbartDto.getLigneKbartDtos()); - } catch (IOException | CsvRequiredFieldEmptyException | CsvDataTypeMismatchException e) { - throw new RuntimeException(e); - } - - } - - private String buildHeader(Class clazz) { - return Arrays.stream(clazz.getDeclaredFields()) - .filter(f -> f.getAnnotation(CsvBindByPosition.class) != null - && f.getAnnotation(CsvBindByName.class) != null) - .sorted(Comparator.comparing(f -> f.getAnnotation(CsvBindByPosition.class).position())) - .map(f -> f.getAnnotation(CsvBindByName.class).column()) - .collect(Collectors.joining("\t")) + "\n"; - } - -} diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index d5bfe04..edcc3d7 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -27,8 +27,6 @@ spring.jpa.bacon.hibernate.ddl-auto=none spring.jpa.bacon.show-sql=true spring.sql.bacon.init.mode=never -path.tsvFile=/tmp/ - logging.config=classpath:log4j2-dev.xml logging.level.root=INFO logging.level.fr.abes=DEBUG diff --git a/src/main/resources/application-prod.properties b/src/main/resources/application-prod.properties index af25484..aa83cf4 100644 --- a/src/main/resources/application-prod.properties +++ b/src/main/resources/application-prod.properties @@ -27,8 +27,6 @@ spring.jpa.bacon.hibernate.ddl-auto=none spring.jpa.bacon.show-sql=true spring.sql.bacon.init.mode=never -path.tsvFile=/tmp/ - logging.config=classpath:log4j2-prod.xml logging.level.root=ERROR -logging.level.fr.abes=ERROR \ No newline at end of file +logging.level.fr.abes=ERROR diff --git a/src/main/resources/application-test.properties b/src/main/resources/application-test.properties index b2be49e..a1faf50 100644 --- a/src/main/resources/application-test.properties +++ b/src/main/resources/application-test.properties @@ -27,8 +27,6 @@ spring.jpa.bacon.hibernate.ddl-auto=none spring.jpa.bacon.show-sql=true spring.sql.bacon.init.mode=never -path.tsvFile=/tmp/ - logging.config=classpath:log4j2-test.xml logging.level.root=INFO logging.level.fr.abes=INFO