From b4d2b9ff70e463179da0c0ca138fe44337dba204 Mon Sep 17 00:00:00 2001 From: SamuelQuetin Date: Thu, 10 Oct 2024 09:29:05 +0200 Subject: [PATCH 1/8] Fix mail si volumineux + fix objet mail --- .../fr/abes/logskbart/kafka/LogsListener.java | 9 ++-- .../abes/logskbart/service/EmailService.java | 48 ++++++++++++++++++- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/main/java/fr/abes/logskbart/kafka/LogsListener.java b/src/main/java/fr/abes/logskbart/kafka/LogsListener.java index 2e21a79..3aa64b3 100644 --- a/src/main/java/fr/abes/logskbart/kafka/LogsListener.java +++ b/src/main/java/fr/abes/logskbart/kafka/LogsListener.java @@ -145,12 +145,15 @@ private void createFileBad(String filename, Integer nbRun) throws IOException { Path pathOfLog = Path.of("tempLog" + File.separator + filename.replace(".tsv", ".log")); log.info("Suppression de " + pathOfLog); Files.deleteIfExists(pathOfLog); - - emailService.sendMailWithAttachment(filename, pathOfBadLocal); + long tailleDixMo = 10 * 1024 * 1024; + if( pathOfBadFinal.toFile().length() < tailleDixMo) { + emailService.sendMailWithAttachment(filename, pathOfBadLocal); + } else { + emailService.sendEmail(filename, "Le fichier est trop volumineux, retrouvez le sur le chemin : /applis/bacon/toLoad/"+filename.replace(".tsv", ".bad")); + } log.info("Suppression de " + pathOfBadLocal + " en local"); Files.deleteIfExists(pathOfBadLocal); } - } } diff --git a/src/main/java/fr/abes/logskbart/service/EmailService.java b/src/main/java/fr/abes/logskbart/service/EmailService.java index 83d36cd..80e241f 100644 --- a/src/main/java/fr/abes/logskbart/service/EmailService.java +++ b/src/main/java/fr/abes/logskbart/service/EmailService.java @@ -11,9 +11,14 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.converter.StringHttpMessageConverter; import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; import java.io.*; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -30,10 +35,18 @@ public class EmailService { @Value("${spring.profiles.active}") private String env; + public void sendEmail(String packageName, String message) { + // Création du mail + String requestJson = mailToJSON(this.recipient, "[KBART2BACON : erreurs]" + getTag() + " " + packageName, message); + + // Envoi du message par mail + sendMail(requestJson); + } + public void sendMailWithAttachment(String packageName, Path mailAttachmentPath) { try { // Création du mail - String requestJson = mailToJSON(this.recipient, "[CONVERGENCE]["+env.toUpperCase()+"] Log(s) d'erreur de " + packageName, ""); + String requestJson = mailToJSON(this.recipient, "[KBART2BACON : erreurs]" + getTag() + " " + packageName, "/applis/bacon/toLoad/"+mailAttachmentPath.getFileName()); // Récupération du fichier File file = mailAttachmentPath.toFile(); @@ -81,6 +94,31 @@ protected void sendMailWithFile(String requestJson, File f) { } } + protected void sendMail(String requestJson) { + RestTemplate restTemplate = new RestTemplate(); //appel ws qui envoie le mail + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + + org.springframework.http.HttpEntity entity = new org.springframework.http.HttpEntity<>(requestJson, headers); + + restTemplate.getMessageConverters() + .add(0, new StringHttpMessageConverter(StandardCharsets.UTF_8)); + + try { + restTemplate.postForObject(url + "htmlMail/", entity, String.class); //appel du ws avec + } catch (Exception e) { + log.warn("Erreur dans l'envoi du mail d'erreur Sudoc" + e); + } + // Création du l'adresse du ws d'envoi de mails + HttpPost mail = new HttpPost(this.url + "htmlMail/"); + + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + httpClient.execute(mail); + } catch (IOException e) { + log.warn("Erreur lors de l'envoi du mail. " + e); + } + } + protected String mailToJSON(String to, String subject, String text) { String json = ""; ObjectMapper mapper = new ObjectMapper(); @@ -98,4 +136,12 @@ protected String mailToJSON(String to, String subject, String text) { } return json; } + + private String getTag(){ + if(env.equalsIgnoreCase("PROD")){ + return ""; + } else { + return "[" + env.toUpperCase() + "]"; + } + } } From e70e39030530bd946be68210dd2a6d9e70c1e3e6 Mon Sep 17 00:00:00 2001 From: SamuelQuetin Date: Mon, 14 Oct 2024 15:24:30 +0200 Subject: [PATCH 2/8] Ajout cas ou nbRun = null --- src/main/java/fr/abes/logskbart/kafka/LogsListener.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/fr/abes/logskbart/kafka/LogsListener.java b/src/main/java/fr/abes/logskbart/kafka/LogsListener.java index 3aa64b3..44e9fda 100644 --- a/src/main/java/fr/abes/logskbart/kafka/LogsListener.java +++ b/src/main/java/fr/abes/logskbart/kafka/LogsListener.java @@ -65,6 +65,9 @@ public void listenInfoKbart2KafkaAndErrorKbart2Kafka(ConsumerRecord 1) ? key[1] : "-1") )); Integer nbRun = service.getLastNbRun(logKbart.getPackageName()); + if(nbRun == null){ + nbRun = 0; + } if(logKbart.getMessage().contains("Debut envois kafka de :")){ nbRun++; } From 843ebada450b38b42196c119fb6309336b5f1c3a Mon Sep 17 00:00:00 2001 From: SamuelQuetin Date: Tue, 15 Oct 2024 10:12:37 +0200 Subject: [PATCH 3/8] Modif config --- .../fr/abes/logskbart/configuration/KafkaConfig.java | 11 ++++++----- .../java/fr/abes/logskbart/service/EmailService.java | 4 +++- src/main/resources/application-dev.properties | 2 +- src/main/resources/application-prod.properties | 2 +- src/main/resources/application-test.properties | 2 +- src/main/resources/application.properties | 5 ----- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java b/src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java index a457358..bbd95b3 100644 --- a/src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java +++ b/src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java @@ -13,24 +13,25 @@ import java.sql.Timestamp; import java.util.HashMap; import java.util.Map; +import java.util.UUID; @Configuration @EnableKafka public class KafkaConfig { - @Value("${spring.kafka.consumer.bootstrap-servers}") + @Value("${abes.kafka.bootstrap-servers}") private String bootstrapAddress; - @Value("${topic.groupid.source}") - private String groupId; - @Bean public ConsumerFactory consumerLogsFactory() { Map props = new HashMap<>(); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress); - props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId); + props.put(ConsumerConfig.GROUP_INSTANCE_ID_CONFIG,("SchedulerCoordinator"+ UUID.randomUUID())); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true); + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 10); + props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 60000); return new DefaultKafkaConsumerFactory<>(props); } diff --git a/src/main/java/fr/abes/logskbart/service/EmailService.java b/src/main/java/fr/abes/logskbart/service/EmailService.java index 80e241f..e4354ba 100644 --- a/src/main/java/fr/abes/logskbart/service/EmailService.java +++ b/src/main/java/fr/abes/logskbart/service/EmailService.java @@ -41,6 +41,8 @@ public void sendEmail(String packageName, String message) { // Envoi du message par mail sendMail(requestJson); + + log.info("L'email a été correctement envoyé à " + recipient); } public void sendMailWithAttachment(String packageName, Path mailAttachmentPath) { @@ -57,7 +59,7 @@ public void sendMailWithAttachment(String packageName, Path mailAttachmentPath) // Suppression du fichier temporaire Files.deleteIfExists(mailAttachmentPath); - log.info("L'email a été correctement envoyé à " + recipient); + log.info("L'email avec PJ a été correctement envoyé à " + recipient); } catch (IOException e) { throw new RuntimeException(e); diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index fb3edd2..80a0b60 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -1,5 +1,5 @@ # Consumer properties -spring.kafka.consumer.bootstrap-servers= +abes.kafka.bootstrap-servers= abes.kafka.concurrency.nbThread= # Properties defined from .env on server #ignore resolution error diff --git a/src/main/resources/application-prod.properties b/src/main/resources/application-prod.properties index 34b174a..315f249 100644 --- a/src/main/resources/application-prod.properties +++ b/src/main/resources/application-prod.properties @@ -1,5 +1,5 @@ # Consumer properties -spring.kafka.consumer.bootstrap-servers= +abes.kafka.bootstrap-servers= abes.kafka.concurrency.nbThread= # Base Postgres diff --git a/src/main/resources/application-test.properties b/src/main/resources/application-test.properties index 4e98d54..0409db2 100644 --- a/src/main/resources/application-test.properties +++ b/src/main/resources/application-test.properties @@ -1,5 +1,5 @@ # Consumer properties -spring.kafka.consumer.bootstrap-servers= +abes.kafka.bootstrap-servers= abes.kafka.concurrency.nbThread= # Properties defined from .env on server #ignore resolution error diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index fbf49b5..5eea022 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -14,11 +14,6 @@ server.port=8082 # Configuration des logs log4j2.logdir=logs -# Common Kafka Properties -spring.kafka.consumer.key-serializer=org.apache.kafka.common.serialization.StringSerializer -spring.kafka.consumer.value-serializer=org.apache.kafka.common.serialization.StringSerializer -spring.kafka.consumer.max-poll-records=10000 - # Topic Kafka topic.name.source.error=bacon.logs.toload # Pour la partie consumer : insertion dans kafka de la ligne kbart avec bestppn From 9c139664a4677aa6a8239a67ac7a5652c299b818 Mon Sep 17 00:00:00 2001 From: pierre-maraval Date: Mon, 21 Oct 2024 15:48:21 +0200 Subject: [PATCH 4/8] =?UTF-8?q?FIX=20:=20Suppression=20conf=20log4j=20obso?= =?UTF-8?q?l=C3=A8te=20Mise=20en=20place=20lecture=20globale=20dans=20kafk?= =?UTF-8?q?a=20avant=20d'envoyer=20vers=20stockage=20Changement=20syst?= =?UTF-8?q?=C3=A8me=20de=20stockage=20de=20postgres=20vers=20ES?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 38 ++--- .../configuration/AbstractConfig.java | 25 ---- .../logskbart/configuration/KafkaConfig.java | 11 +- .../logskbart/configuration/LogsBdConfig.java | 54 ------- .../configuration/LogsBdConfiguration.java | 8 -- .../logskbart/controller/LogsController.java | 12 +- .../dto/{LogDto.java => LogWebDto.java} | 4 +- .../fr/abes/logskbart/entity/LogKbart.java | 47 +++---- .../fr/abes/logskbart/kafka/LogsListener.java | 78 +++++++---- .../abes/logskbart/kafka/WorkInProgress.java | 30 ++++ .../repository/LogKbartRepository.java | 12 +- .../abes/logskbart/service/LogsService.java | 18 +-- .../fr/abes/logskbart/utils/LogsMapper.java | 4 +- src/main/resources/application-dev.properties | 13 +- .../resources/application-prod.properties | 13 +- .../resources/application-test.properties | 13 +- src/main/resources/application.properties | 4 +- src/main/resources/log4j2-dev.xml | 132 ------------------ src/main/resources/log4j2-prod.xml | 132 ------------------ src/main/resources/log4j2-test.xml | 132 ------------------ src/main/resources/log4j2.xml | 19 +++ 21 files changed, 165 insertions(+), 634 deletions(-) delete mode 100644 src/main/java/fr/abes/logskbart/configuration/AbstractConfig.java delete mode 100644 src/main/java/fr/abes/logskbart/configuration/LogsBdConfig.java delete mode 100644 src/main/java/fr/abes/logskbart/configuration/LogsBdConfiguration.java rename src/main/java/fr/abes/logskbart/dto/{LogDto.java => LogWebDto.java} (85%) create mode 100644 src/main/java/fr/abes/logskbart/kafka/WorkInProgress.java delete mode 100644 src/main/resources/log4j2-dev.xml delete mode 100644 src/main/resources/log4j2-prod.xml delete mode 100644 src/main/resources/log4j2-test.xml create mode 100644 src/main/resources/log4j2.xml diff --git a/pom.xml b/pom.xml index 6ee58b6..31b7e5d 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.springframework.boot spring-boot-starter-parent - 3.0.4 + 3.2.8 fr.abes @@ -76,28 +76,19 @@ org.springframework.kafka spring-kafka - - - - org.springframework.boot - spring-boot-starter-data-jpa - org.springframework.boot - spring-boot-starter-logging + org.scala-lang + scala-library + - org.postgresql - postgresql - - - - com.oracle.database.jdbc - ojdbc8 - 21.9.0.0 + org.springframework.boot + spring-boot-starter-data-elasticsearch + org.springframework.boot @@ -135,11 +126,6 @@ - - org.projectlombok - lombok - true - org.apache.httpcomponents httpmime @@ -150,7 +136,7 @@ org.springdoc springdoc-openapi-starter-webmvc-ui - 2.1.0 + 2.3.0 @@ -161,7 +147,7 @@ com.fasterxml.jackson.dataformat jackson-dataformat-xml - 2.12.6 + 2.17.0 @@ -171,9 +157,9 @@ test - org.springframework.kafka - spring-kafka-test - test + org.projectlombok + lombok + provided diff --git a/src/main/java/fr/abes/logskbart/configuration/AbstractConfig.java b/src/main/java/fr/abes/logskbart/configuration/AbstractConfig.java deleted file mode 100644 index ec60872..0000000 --- a/src/main/java/fr/abes/logskbart/configuration/AbstractConfig.java +++ /dev/null @@ -1,25 +0,0 @@ -package fr.abes.logskbart.configuration; - -import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; -import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; - -import java.util.HashMap; - -public abstract class AbstractConfig { - protected void configHibernate(LocalContainerEntityManagerFactoryBean em, String platform, boolean showsql, String dialect, String ddlAuto, boolean generateDdl, String initMode) { - HibernateJpaVendorAdapter vendorAdapter - = new HibernateJpaVendorAdapter(); - vendorAdapter.setGenerateDdl(generateDdl); - vendorAdapter.setShowSql(showsql); - vendorAdapter.setDatabasePlatform(platform); - em.setJpaVendorAdapter(vendorAdapter); - HashMap properties = new HashMap<>(); - properties.put("hibernate.format_sql", true); - properties.put("hibernate.hbm2ddl.auto", ddlAuto); - properties.put("hibernate.dialect", dialect); - properties.put("logging.level.org.hibernate", "DEBUG"); - properties.put("hibernate.type", "trace"); - properties.put("spring.sql.init.mode", initMode); - em.setJpaPropertyMap(properties); - } -} diff --git a/src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java b/src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java index bbd95b3..3e898c5 100644 --- a/src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java +++ b/src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java @@ -1,5 +1,6 @@ package fr.abes.logskbart.configuration; +import fr.abes.logskbart.kafka.WorkInProgress; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.serialization.StringDeserializer; import org.springframework.beans.factory.annotation.Value; @@ -9,11 +10,13 @@ import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.listener.ContainerProperties; import java.sql.Timestamp; import java.util.HashMap; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; @Configuration @EnableKafka @@ -30,8 +33,8 @@ public ConsumerFactory consumerLogsFactory() { props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true); - props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 10); - props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 60000); + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 100); + props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 30000); return new DefaultKafkaConsumerFactory<>(props); } @@ -45,7 +48,7 @@ public ConsumerFactory consumerLogsFactory() { } @Bean - public Map lastTimeStampByFilename() { - return new HashMap<>(); + public Map workInProgressMap() { + return new ConcurrentHashMap<>(); } } diff --git a/src/main/java/fr/abes/logskbart/configuration/LogsBdConfig.java b/src/main/java/fr/abes/logskbart/configuration/LogsBdConfig.java deleted file mode 100644 index 418a6d3..0000000 --- a/src/main/java/fr/abes/logskbart/configuration/LogsBdConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -package fr.abes.logskbart.configuration; - -import lombok.NoArgsConstructor; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.jdbc.DataSourceBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.jpa.repository.config.EnableJpaRepositories; -import org.springframework.orm.jpa.JpaTransactionManager; -import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; -import org.springframework.transaction.PlatformTransactionManager; - -import javax.sql.DataSource; - -@Configuration -@EnableJpaRepositories(entityManagerFactoryRef = "logsEntityManager", transactionManagerRef = "logsTransactionManager", basePackages = "fr.abes.logskbart.repository") -@NoArgsConstructor -@LogsBdConfiguration -public class LogsBdConfig extends AbstractConfig { - @Value("${spring.jpa.logsdb.show-sql}") - protected boolean showsql; - @Value("${spring.jpa.logsdb.properties.hibernate.dialect}") - protected String dialect; - @Value("${spring.jpa.logsdb.hibernate.ddl-auto}") - protected String ddlAuto; - @Value("${spring.jpa.logsdb.database-platform}") - protected String platform; - @Value("${spring.jpa.logsdb.generate-ddl}") - protected boolean generateDdl; - @Value("${spring.sql.logsdb.init.mode}") - protected String initMode; - - @Bean - @ConfigurationProperties(prefix = "spring.datasource.logsdb") - public DataSource logsDataSource() { return DataSourceBuilder.create().build(); } - - @Bean - public LocalContainerEntityManagerFactoryBean logsEntityManager() { - LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean(); - em.setDataSource(logsDataSource()); - em.setPackagesToScan(new String[]{"fr.abes.logskbart.entity"}); - configHibernate(em, platform, showsql, dialect, ddlAuto, generateDdl, initMode); - return em; - } - - @Bean - public PlatformTransactionManager logsTransactionManager(@Qualifier("logsEntityManager") LocalContainerEntityManagerFactoryBean entityManagerFactory) { - final JpaTransactionManager transactionManager = new JpaTransactionManager(); - transactionManager.setEntityManagerFactory(entityManagerFactory.getObject()); - return transactionManager; - } -} diff --git a/src/main/java/fr/abes/logskbart/configuration/LogsBdConfiguration.java b/src/main/java/fr/abes/logskbart/configuration/LogsBdConfiguration.java deleted file mode 100644 index 7e5310b..0000000 --- a/src/main/java/fr/abes/logskbart/configuration/LogsBdConfiguration.java +++ /dev/null @@ -1,8 +0,0 @@ -package fr.abes.logskbart.configuration; - -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -@Retention(RetentionPolicy.RUNTIME) -public @interface LogsBdConfiguration { -} diff --git a/src/main/java/fr/abes/logskbart/controller/LogsController.java b/src/main/java/fr/abes/logskbart/controller/LogsController.java index 233afa3..8a353d5 100644 --- a/src/main/java/fr/abes/logskbart/controller/LogsController.java +++ b/src/main/java/fr/abes/logskbart/controller/LogsController.java @@ -1,7 +1,7 @@ package fr.abes.logskbart.controller; import fr.abes.logskbart.dto.LigneLogDto; -import fr.abes.logskbart.dto.LogDto; +import fr.abes.logskbart.dto.LogWebDto; import fr.abes.logskbart.service.LogsService; import fr.abes.logskbart.utils.UtilsMapper; import io.swagger.v3.oas.annotations.Operation; @@ -34,18 +34,18 @@ public LogsController(LogsService service, UtilsMapper mapper) { summary = "Transfer kafka to PostgreSQL DB", description = "Retrieves kbart load logs from a Kafka bus and stores them in a DB for later availability", responses = { - @ApiResponse( responseCode = "200", description = "The request was successful.", content = { @Content(schema = @Schema(implementation = LogDto.class), mediaType = "application/json") } ), + @ApiResponse( responseCode = "200", description = "The request was successful.", content = { @Content(schema = @Schema(implementation = LogWebDto.class), mediaType = "application/json") } ), @ApiResponse( responseCode = "400", description = "An element of the query is badly formulated.", content = { @Content(schema = @Schema()) } ), @ApiResponse( responseCode = "500", description = "An internal server error interrupted processing.", content = { @Content(schema = @Schema()) } ), } ) @GetMapping("/logs/{filename}/{date}") - public LogDto getLogsFromPackageAndDate(@PathVariable String filename, @PathVariable String date) throws ParseException { + public LogWebDto getLogsFromPackageAndDate(@PathVariable String filename, @PathVariable String date) throws ParseException { SimpleDateFormat format = new SimpleDateFormat("ddMMyyyy"); Date dateAnalyse = format.parse(date); - LogDto logDto = new LogDto(filename, date); + LogWebDto logWebDto = new LogWebDto(filename, date); List lignes = mapper.mapList(service.getLogKbartForPackage(filename, dateAnalyse), LigneLogDto.class); - logDto.addLignes(lignes); - return logDto; + logWebDto.addLignes(lignes); + return logWebDto; } } diff --git a/src/main/java/fr/abes/logskbart/dto/LogDto.java b/src/main/java/fr/abes/logskbart/dto/LogWebDto.java similarity index 85% rename from src/main/java/fr/abes/logskbart/dto/LogDto.java rename to src/main/java/fr/abes/logskbart/dto/LogWebDto.java index b4d4842..ff38bf1 100644 --- a/src/main/java/fr/abes/logskbart/dto/LogDto.java +++ b/src/main/java/fr/abes/logskbart/dto/LogWebDto.java @@ -8,12 +8,12 @@ @Getter @Setter -public class LogDto { +public class LogWebDto { public String filename; public String date; List ligneLogs; - public LogDto(String filename, String date) { + public LogWebDto(String filename, String date) { this.filename = filename; this.date = date; this.ligneLogs = new ArrayList<>(); diff --git a/src/main/java/fr/abes/logskbart/entity/LogKbart.java b/src/main/java/fr/abes/logskbart/entity/LogKbart.java index 650fd92..16a6ca1 100644 --- a/src/main/java/fr/abes/logskbart/entity/LogKbart.java +++ b/src/main/java/fr/abes/logskbart/entity/LogKbart.java @@ -1,62 +1,57 @@ package fr.abes.logskbart.entity; import fr.abes.logskbart.utils.Level; -import jakarta.persistence.*; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; +import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.annotations.Document; +import org.springframework.data.elasticsearch.annotations.Field; import java.io.Serializable; import java.util.Date; -@Entity -@Table(name = "LOGKBART") -@Getter -@Setter -@NoArgsConstructor +@Document(indexName = "logkbart") +@Data @Slf4j public class LogKbart implements Serializable { @Id - @Column(name = "ID") - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; + @Field(name = "ID") + private String id; - @Column(name = "PACKAGE_NAME") + @Field(name = "PACKAGE_NAME") private String packageName; - @Column(name = "TIMESTAMP") + @Field(name = "TIMESTAMP") private Date timestamp; - @Column(name = "THREAD") + @Field(name = "THREAD") private String thread; - @Column(name = "LEVEL") - @Enumerated(EnumType.STRING) - private Level level; + @Field(name = "LEVEL") + private String level; - @Column(name = "LOGGER_NAME") + @Field(name = "LOGGER_NAME") private String loggerName; - @Column(name = "message", length = 2048) + @Field(name = "MESSAGE") private String message; - @Column(name = "END_OF_BATCH") + @Field(name = "END_OF_BATCH") private boolean endOfBatch; - @Column(name = "LOGGER_FQCN") + @Field(name = "LOGGER_FQCN") private String loggerFqcn; - @Column(name = "THREAD_ID") + @Field(name = "THREAD_ID") private Integer threadId; - @Column(name = "THREAD_PRIORITY") + @Field(name = "THREAD_PRIORITY") private Integer threadPriority; - @Column(name = "NB_LINE", nullable = false) + @Field(name = "NB_LINE") private Integer nbLine; - @Column(name = "NB_RUN", nullable = false) + @Field(name = "NB_RUN") private Integer nbRun = 0; @Override diff --git a/src/main/java/fr/abes/logskbart/kafka/LogsListener.java b/src/main/java/fr/abes/logskbart/kafka/LogsListener.java index 44e9fda..c186aa6 100644 --- a/src/main/java/fr/abes/logskbart/kafka/LogsListener.java +++ b/src/main/java/fr/abes/logskbart/kafka/LogsListener.java @@ -9,6 +9,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.Acknowledgment; import org.springframework.stereotype.Service; import java.io.File; @@ -18,11 +19,11 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; -import java.sql.Timestamp; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.Date; import java.util.List; +import java.util.Map; @Slf4j @@ -37,11 +38,14 @@ public class LogsListener { private final EmailService emailService; - public LogsListener(ObjectMapper mapper, UtilsMapper logsMapper, LogsService service, EmailService emailService) { + private final Map workInProgressMap; + + public LogsListener(ObjectMapper mapper, UtilsMapper logsMapper, LogsService service, EmailService emailService, Map workInProgressMap) { this.mapper = mapper; this.logsMapper = logsMapper; this.service = service; this.emailService = emailService; + this.workInProgressMap = workInProgressMap; } @@ -51,44 +55,58 @@ public LogsListener(ObjectMapper mapper, UtilsMapper logsMapper, LogsService ser * @param message le message kafka * @throws IOException exception levée */ - @KafkaListener(topics = {"${topic.name.source.error}"}, groupId = "${topic.groupid.source}", containerFactory = "kafkaLogsListenerContainerFactory", concurrency = "${abes.kafka.concurrency.nbThread}") + @KafkaListener(topics = {"${topic.name.source.error}"}, groupId = "${topic.groupid.source}", containerFactory = "kafkaLogsListenerContainerFactory") public void listenInfoKbart2KafkaAndErrorKbart2Kafka(ConsumerRecord message) throws IOException { LogKbartDto dto = mapper.readValue(message.value(), LogKbartDto.class); - LogKbart logKbart = logsMapper.map(dto, LogKbart.class); - // recuperation de l'heure a laquelle le message a ete envoye String[] key = message.key().split(";"); - - Timestamp currentTimestamp = new Timestamp(message.timestamp()); - logKbart.setTimestamp(new Date(currentTimestamp.getTime())); - logKbart.setPackageName(key[0]); - logKbart.setNbLine(Integer.parseInt(((key.length > 1) ? key[1] : "-1") )); - - Integer nbRun = service.getLastNbRun(logKbart.getPackageName()); - if(nbRun == null){ - nbRun = 0; + String packageName = key[0]; + if (!this.workInProgressMap.containsKey(packageName)) { + //nouveau fichier trouvé dans le topic, on initialise les variables partagées + log.debug("Nouveau package identifié : " + packageName); + workInProgressMap.put(packageName, new WorkInProgress()); } - if(logKbart.getMessage().contains("Debut envois kafka de :")){ - nbRun++; + workInProgressMap.get(packageName).addMessage(dto); + if (!packageName.contains("ctx:package") && !packageName.contains("_FORCE")) { + if ((dto.getMessage().contains("Traitement terminé pour fichier " + packageName)) || (dto.getMessage().contains("Traitement refusé du fichier " + packageName))) { + log.debug("Commit les datas pour fichier " + packageName); + Integer nbLine = Integer.parseInt(((key.length > 1) ? key[1] : "-1")); + Integer nbRun = commitDatas(message.timestamp(), packageName, nbLine); + createFileBad(packageName, nbRun); + workInProgressMap.remove(packageName); + } } - logKbart.setNbRun(nbRun); + } - logKbart.log(); - // Inscrit l'entity en BDD - service.save(logKbart); + private Integer commitDatas(long timeStamp, String packageName, Integer nbLine) { + long startTime = System.currentTimeMillis(); + log.debug("Debut Commit datas pour fichier " + packageName); + List logskbart = logsMapper.mapList(workInProgressMap.get(packageName).getMessages(), LogKbart.class); + int nbRun = service.getLastNbRun(packageName) + 1; + log.debug("NbRun: " + nbRun); + saveDatas(timeStamp, packageName, nbLine, logskbart, nbRun); + log.debug("datas saved pour fichier " + packageName); + long endTime = System.currentTimeMillis(); + double executionTime = (double) (endTime - startTime) / 1000; + log.debug("Execution time: " + executionTime); + return nbRun; + } - if (!logKbart.getPackageName().contains("ctx:package") && !logKbart.getPackageName().contains("_FORCE")) { - if( (logKbart.getMessage().contains("Traitement terminé pour fichier " + logKbart.getPackageName())) || (logKbart.getMessage().contains("Traitement refusé du fichier " + logKbart.getPackageName())) ) { - createFileBad(logKbart.getPackageName(),nbRun); - } - } + private void saveDatas(long timeStamp, String packageName, Integer nbLine, List logskbart, int nbRun) { + logskbart.forEach(logKbart -> { + logKbart.setNbRun(nbRun); + logKbart.setTimestamp(new Date(timeStamp)); + logKbart.setPackageName(packageName); + logKbart.setNbLine(nbLine); + }); + service.saveAll(logskbart); } public void deleteOldLocalTempLog() throws IOException { File dirToCheck = new File("tempLogLocal"); File[] listeFilesTempLogLocal = dirToCheck.listFiles(); if (listeFilesTempLogLocal != null) { - for (File fileToCheck: listeFilesTempLogLocal) { + for (File fileToCheck : listeFilesTempLogLocal) { BasicFileAttributes basicFileAttributes = Files.readAttributes(fileToCheck.toPath(), BasicFileAttributes.class); if (basicFileAttributes.isRegularFile()) { String nameFile = String.valueOf(fileToCheck); @@ -104,9 +122,9 @@ public void deleteOldLocalTempLog() throws IOException { } private void createFileBad(String filename, Integer nbRun) throws IOException { - List logKbartList = service.getErrorLogKbartByPackageAndNbRun(filename,nbRun); + List logKbartList = service.getErrorLogKbartByPackageAndNbRun(filename, nbRun); Path tempPath = Path.of("tempLogLocal"); - if(!Files.exists(tempPath)) { + if (!Files.exists(tempPath)) { Files.createDirectory(tempPath); } Path pathOfBadLocal = Path.of("tempLogLocal" + File.separator + filename.replace(".tsv", ".bad")); @@ -149,10 +167,10 @@ private void createFileBad(String filename, Integer nbRun) throws IOException { log.info("Suppression de " + pathOfLog); Files.deleteIfExists(pathOfLog); long tailleDixMo = 10 * 1024 * 1024; - if( pathOfBadFinal.toFile().length() < tailleDixMo) { + if (pathOfBadFinal.toFile().length() < tailleDixMo) { emailService.sendMailWithAttachment(filename, pathOfBadLocal); } else { - emailService.sendEmail(filename, "Le fichier est trop volumineux, retrouvez le sur le chemin : /applis/bacon/toLoad/"+filename.replace(".tsv", ".bad")); + emailService.sendEmail(filename, "Le fichier est trop volumineux, retrouvez le sur le chemin : /applis/bacon/toLoad/" + filename.replace(".tsv", ".bad")); } log.info("Suppression de " + pathOfBadLocal + " en local"); diff --git a/src/main/java/fr/abes/logskbart/kafka/WorkInProgress.java b/src/main/java/fr/abes/logskbart/kafka/WorkInProgress.java new file mode 100644 index 0000000..226ca31 --- /dev/null +++ b/src/main/java/fr/abes/logskbart/kafka/WorkInProgress.java @@ -0,0 +1,30 @@ +package fr.abes.logskbart.kafka; + +import fr.abes.logskbart.dto.LogKbartDto; +import fr.abes.logskbart.entity.LogKbart; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@Getter +@Setter +@Slf4j +public class WorkInProgress { + private List messages; + + private Timestamp timestamp; + + public WorkInProgress() { + this.messages = Collections.synchronizedList(new ArrayList<>()); + } + + public void addMessage(LogKbartDto message) { + this.messages.add(message); + } + +} diff --git a/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java b/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java index 24c5716..1b35e98 100644 --- a/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java +++ b/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java @@ -1,21 +1,17 @@ package fr.abes.logskbart.repository; -import fr.abes.logskbart.configuration.LogsBdConfiguration; import fr.abes.logskbart.entity.LogKbart; -import fr.abes.logskbart.utils.Level; -import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; import org.springframework.stereotype.Repository; import java.util.Date; import java.util.List; -import java.util.Optional; @Repository -@LogsBdConfiguration -public interface LogKbartRepository extends JpaRepository { +public interface LogKbartRepository extends ElasticsearchRepository { List findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(String filename, Date debut, Date fin); - List findAllByPackageNameAndNbRunAndLevelOrderByNbLineAscTimestampAsc(String filename, Integer nbRun, Level level); + List findAllByPackageNameAndNbRunAndLevelOrderByNbLineAscTimestampAsc(String filename, Integer nbRun, String level); - Optional getFirstByPackageNameOrderByNbRunDesc(String filename); + List findByPackageNameOrderByNbRunDesc(String filename); } diff --git a/src/main/java/fr/abes/logskbart/service/LogsService.java b/src/main/java/fr/abes/logskbart/service/LogsService.java index 0518988..e06ad17 100644 --- a/src/main/java/fr/abes/logskbart/service/LogsService.java +++ b/src/main/java/fr/abes/logskbart/service/LogsService.java @@ -16,6 +16,7 @@ public class LogsService { private final LogKbartRepository repository; + public LogsService(LogKbartRepository repository) { this.repository = repository; } @@ -29,20 +30,21 @@ public List getLogKbartForPackage(String packageName, Date date) { return repository.findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(packageName, dateChargement.getTime(), dateFin.getTime()); } - public LogKbart save(LogKbart logKbart) { - return repository.save(logKbart); + public void saveAll(List logKbarts) { + repository.saveAll(logKbarts); } public Integer getLastNbRun(String packageName) { - Optional logKbart = repository.getFirstByPackageNameOrderByNbRunDesc(packageName); - if(logKbart.isPresent()) { - return logKbart.get().getNbRun(); - } else { - return 0; + List logskbart = repository.findByPackageNameOrderByNbRunDesc(packageName); + Optional logKbart = logskbart.stream().findFirst(); + if (logKbart.isPresent()) { + Integer nbRun = logKbart.get().getNbRun(); + return (nbRun != null) ? nbRun : 0; } + return 0; } public List getErrorLogKbartByPackageAndNbRun(String packageName, Integer nbRun) { - return repository.findAllByPackageNameAndNbRunAndLevelOrderByNbLineAscTimestampAsc(packageName,nbRun, Level.ERROR); + return repository.findAllByPackageNameAndNbRunAndLevelOrderByNbLineAscTimestampAsc(packageName,nbRun, String.valueOf(Level.ERROR)); } } diff --git a/src/main/java/fr/abes/logskbart/utils/LogsMapper.java b/src/main/java/fr/abes/logskbart/utils/LogsMapper.java index c914bab..acce95e 100644 --- a/src/main/java/fr/abes/logskbart/utils/LogsMapper.java +++ b/src/main/java/fr/abes/logskbart/utils/LogsMapper.java @@ -25,7 +25,7 @@ public void converterInfoBaconDtoToLogKbart() { public LogKbart convert(MappingContext context) { LogKbartDto source = context.getSource(); LogKbart target = new LogKbart(); - target.setLevel(Level.valueOf(source.getLevel())); + target.setLevel(source.getLevel()); target.setMessage(source.getMessage()); target.setThread(source.getThread()); target.setLoggerFqcn(source.getLoggerFqcn()); @@ -46,7 +46,7 @@ public void converterLogKbartToLogDto() { public LigneLogDto convert(MappingContext context) { LogKbart source = context.getSource(); LigneLogDto target = new LigneLogDto(); - target.setLevel(source.getLevel().toString()); + target.setLevel(source.getLevel()); target.setMessage(source.getMessage()); target.setNbLine(source.getNbLine()); return target; diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 80a0b60..35204bf 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -3,19 +3,8 @@ abes.kafka.bootstrap-servers= abes.kafka.concurrency.nbThread= # Properties defined from .env on server #ignore resolution error -spring.datasource.logsdb.driver-class-name=org.postgresql.Driver -spring.datasource.logsdb.jdbcurl= -spring.datasource.logsdb.username= -spring.datasource.logsdb.password= +spring.elasticsearch.uris= -spring.jpa.logsdb.database-platform=org.hibernate.dialect.PostgreSQLDialect -spring.jpa.logsdb.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQLDialect -spring.jpa.logsdb.generate-ddl=true -spring.jpa.logsdb.hibernate.ddl-auto=update -spring.jpa.logsdb.show-sql=false -spring.sql.logsdb.init.mode=never - -logging.config=classpath:log4j2-dev.xml logging.level.root=INFO logging.level.fr.abes=DEBUG diff --git a/src/main/resources/application-prod.properties b/src/main/resources/application-prod.properties index 315f249..a511fcf 100644 --- a/src/main/resources/application-prod.properties +++ b/src/main/resources/application-prod.properties @@ -4,20 +4,9 @@ abes.kafka.concurrency.nbThread= # Base Postgres # Properties defined from .env on server #ignore resolution error -spring.datasource.logsdb.driver-class-name=org.postgresql.Driver -spring.datasource.logsdb.jdbcurl= -spring.datasource.logsdb.username= -spring.datasource.logsdb.password= +spring.elasticsearch.uris= -spring.jpa.logsdb.database-platform=org.hibernate.dialect.PostgreSQLDialect -spring.jpa.logsdb.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQLDialect -spring.jpa.logsdb.generate-ddl=true -spring.jpa.logsdb.hibernate.ddl-auto=update -spring.jpa.logsdb.show-sql=false -spring.sql.logsdb.init.mode=never - -logging.config=classpath:log4j2-prod.xml logging.level.root=ERROR logging.level.fr.abes=ERROR diff --git a/src/main/resources/application-test.properties b/src/main/resources/application-test.properties index 0409db2..45482d8 100644 --- a/src/main/resources/application-test.properties +++ b/src/main/resources/application-test.properties @@ -3,19 +3,8 @@ abes.kafka.bootstrap-servers= abes.kafka.concurrency.nbThread= # Properties defined from .env on server #ignore resolution error -spring.datasource.logsdb.driver-class-name=org.postgresql.Driver -spring.datasource.logsdb.jdbcurl= -spring.datasource.logsdb.username= -spring.datasource.logsdb.password= +spring.elasticsearch.uris= -spring.jpa.logsdb.database-platform=org.hibernate.dialect.PostgreSQLDialect -spring.jpa.logsdb.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQLDialect -spring.jpa.logsdb.generate-ddl=true -spring.jpa.logsdb.hibernate.ddl-auto=update -spring.jpa.logsdb.show-sql=false -spring.sql.logsdb.init.mode=never - -logging.config=classpath:log4j2-test.xml logging.level.root=INFO logging.level.fr.abes=INFO diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 5eea022..6078f36 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -11,8 +11,7 @@ application.basedir=@webBaseDir@ # Configuration du serveur Http server.port=8082 -# Configuration des logs -log4j2.logdir=logs +logging.config=classpath:log4j2.xml # Topic Kafka topic.name.source.error=bacon.logs.toload @@ -20,7 +19,6 @@ topic.name.source.error=bacon.logs.toload spring.jpa.open-in-view=false - # SpringDoc (Swagger) logskbart.openapi.url= springdoc.swagger-ui.path=/logskbart-documentation diff --git a/src/main/resources/log4j2-dev.xml b/src/main/resources/log4j2-dev.xml deleted file mode 100644 index e48a0bf..0000000 --- a/src/main/resources/log4j2-dev.xml +++ /dev/null @@ -1,132 +0,0 @@ - - - - - - ${bundle:application:application.basedir}/${bundle:application:log4j2.logdir} - ${bundle:application:application.name} - ${bundle:application:application.name}_error - ${bundle:application:application.name}_debug - - - 100 MB - 1d - 1 - 10 MB - - - - - - - - - - - %5p %d{ISO8601} [%t][%x] %c - %m%n - - - - - - - - - - - - - - - - - - - - - - - - - - - %5p %d{ISO8601} [%t][%x] %c - %m%n - - - - - - - - - - - - - - - - - - - - - - - - - - - %5p %d{ISO8601} [%t][%x] %c - %m%n - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/main/resources/log4j2-prod.xml b/src/main/resources/log4j2-prod.xml deleted file mode 100644 index e48a0bf..0000000 --- a/src/main/resources/log4j2-prod.xml +++ /dev/null @@ -1,132 +0,0 @@ - - - - - - ${bundle:application:application.basedir}/${bundle:application:log4j2.logdir} - ${bundle:application:application.name} - ${bundle:application:application.name}_error - ${bundle:application:application.name}_debug - - - 100 MB - 1d - 1 - 10 MB - - - - - - - - - - - %5p %d{ISO8601} [%t][%x] %c - %m%n - - - - - - - - - - - - - - - - - - - - - - - - - - - %5p %d{ISO8601} [%t][%x] %c - %m%n - - - - - - - - - - - - - - - - - - - - - - - - - - - %5p %d{ISO8601} [%t][%x] %c - %m%n - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/main/resources/log4j2-test.xml b/src/main/resources/log4j2-test.xml deleted file mode 100644 index e48a0bf..0000000 --- a/src/main/resources/log4j2-test.xml +++ /dev/null @@ -1,132 +0,0 @@ - - - - - - ${bundle:application:application.basedir}/${bundle:application:log4j2.logdir} - ${bundle:application:application.name} - ${bundle:application:application.name}_error - ${bundle:application:application.name}_debug - - - 100 MB - 1d - 1 - 10 MB - - - - - - - - - - - %5p %d{ISO8601} [%t][%x] %c - %m%n - - - - - - - - - - - - - - - - - - - - - - - - - - - %5p %d{ISO8601} [%t][%x] %c - %m%n - - - - - - - - - - - - - - - - - - - - - - - - - - - %5p %d{ISO8601} [%t][%x] %c - %m%n - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml new file mode 100644 index 0000000..c582f44 --- /dev/null +++ b/src/main/resources/log4j2.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + From 9ced31780155b11253bf5e831152d2f2ed2a70f9 Mon Sep 17 00:00:00 2001 From: pierre-maraval Date: Tue, 22 Oct 2024 15:58:35 +0200 Subject: [PATCH 5/8] =?UTF-8?q?FIX=20:=20envoi=20des=20messages=20=C3=A0?= =?UTF-8?q?=20ES=20Ajout=20entry=20point=20pour=20cr=C3=A9er=20l'index=20a?= =?UTF-8?q?u=20d=C3=A9marrage=20du=20container=20s'il=20n'existe=20pas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 4 +- docker/docker-entrypoint.sh | 53 ++++++++++++ .../logskbart/configuration/KafkaConfig.java | 18 +++- .../fr/abes/logskbart/entity/LogKbart.java | 15 ++-- .../fr/abes/logskbart/kafka/LogsListener.java | 86 ++++++++++--------- .../abes/logskbart/kafka/WorkInProgress.java | 4 +- .../repository/LogKbartRepository.java | 2 - .../abes/logskbart/service/LogsService.java | 18 +--- src/main/resources/application.properties | 4 +- 9 files changed, 134 insertions(+), 70 deletions(-) create mode 100644 docker/docker-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 51cd1ec..3800047 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,4 +36,6 @@ WORKDIR /app/ COPY --from=build-image /build/target/*.jar /app/logskbart-api.jar ENV TZ=Europe/Paris RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -ENTRYPOINT ["java","-jar","/app/logskbart-api.jar"] +COPY ./docker/docker-entrypoint.sh /docker-entrypoint.sh +RUN chmod +x /docker-entrypoint.sh +ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh new file mode 100644 index 0000000..6a607fd --- /dev/null +++ b/docker/docker-entrypoint.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +export SPRING_ELASTICSEARCH_URIS=${SPRING_ELASTICSEARCH_URIS:='http://localhost:9200'} + +curl -X PUT "$SPRING_ELASTICSEARCH_URIS/logkbart" -H 'Content-Type: application/json' -d' + { + "settings": { + "number_of_shards": 5, + "number_of_replicas": 0 + }, + "mappings": { + "properties": { + "ID": { + "type": "text" + }, + "PACKAGE_NAME": { + "type": "text" + }, + "TIMESTAMP": { + "type": "date" + }, + "THREAD": { + "type": "text" + }, + "LEVEL": { + "type": "text" + }, + "LOGGER_NAME": { + "type": "text" + }, + "MESSAGE": { + "type": "text" + }, + "END_OF_BATCH": { + "type": "boolean" + }, + "LOGGER_FQCN": { + "type": "text" + }, + "THREAD_ID": { + "type": "integer" + }, + "THREAD_PRIORITY": { + "type": "integer" + }, + "NB_LINE": { + "type": "integer" + } + } + } + }' + +java -jar /app/logskbart-api.jar diff --git a/src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java b/src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java index 3e898c5..1bd49d7 100644 --- a/src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java +++ b/src/main/java/fr/abes/logskbart/configuration/KafkaConfig.java @@ -11,18 +11,24 @@ import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import java.sql.Timestamp; import java.util.HashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; @Configuration @EnableKafka public class KafkaConfig { @Value("${abes.kafka.bootstrap-servers}") private String bootstrapAddress; + @Value("${abes.nbThread}") + private int nbThread; @Bean @@ -34,7 +40,7 @@ public ConsumerFactory consumerLogsFactory() { props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true); props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 100); - props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 30000); + props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 60000); return new DefaultKafkaConsumerFactory<>(props); } @@ -51,4 +57,14 @@ public ConsumerFactory consumerLogsFactory() { public Map workInProgressMap() { return new ConcurrentHashMap<>(); } + + @Bean + public Executor executor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(nbThread); + executor.setMaxPoolSize(nbThread); + executor.setQueueCapacity(500); + executor.initialize(); + return executor; + } } diff --git a/src/main/java/fr/abes/logskbart/entity/LogKbart.java b/src/main/java/fr/abes/logskbart/entity/LogKbart.java index 16a6ca1..a5ed3ff 100644 --- a/src/main/java/fr/abes/logskbart/entity/LogKbart.java +++ b/src/main/java/fr/abes/logskbart/entity/LogKbart.java @@ -8,12 +8,14 @@ import org.springframework.data.elasticsearch.annotations.Field; import java.io.Serializable; +import java.util.Comparator; import java.util.Date; +import java.util.Objects; @Document(indexName = "logkbart") @Data @Slf4j -public class LogKbart implements Serializable { +public class LogKbart implements Serializable, Comparable { @Id @Field(name = "ID") private String id; @@ -51,8 +53,6 @@ public class LogKbart implements Serializable { @Field(name = "NB_LINE") private Integer nbLine; - @Field(name = "NB_RUN") - private Integer nbRun = 0; @Override public String toString() { @@ -64,12 +64,17 @@ public String toString() { ", message='" + message + '\'' + ", loggerFqcn='" + loggerFqcn + '\'' + ", nbLine='" + nbLine + '\'' + - ", nbRun='" + nbRun + '\'' + - '}'; } public void log(){ log.debug( this.level +" : " + this); } + + @Override + public int compareTo(LogKbart logKbart) { + if (!Objects.equals(this.nbLine, logKbart.getNbLine())) + return Integer.compare(this.nbLine, logKbart.getNbLine()); + return this.timestamp.compareTo(logKbart.getTimestamp()); + } } diff --git a/src/main/java/fr/abes/logskbart/kafka/LogsListener.java b/src/main/java/fr/abes/logskbart/kafka/LogsListener.java index c186aa6..a23c884 100644 --- a/src/main/java/fr/abes/logskbart/kafka/LogsListener.java +++ b/src/main/java/fr/abes/logskbart/kafka/LogsListener.java @@ -8,8 +8,10 @@ import fr.abes.logskbart.utils.UtilsMapper; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.springframework.beans.factory.annotation.Value; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.support.Acknowledgment; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import java.io.File; @@ -21,14 +23,18 @@ import java.nio.file.attribute.BasicFileAttributes; import java.time.LocalDateTime; import java.time.ZoneId; -import java.util.Date; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; +import java.util.stream.IntStream; @Slf4j @Service public class LogsListener { + @Value("${elasticsearch.max-packet-size}") + private int maxPacketSize; private final ObjectMapper mapper; @@ -40,12 +46,15 @@ public class LogsListener { private final Map workInProgressMap; - public LogsListener(ObjectMapper mapper, UtilsMapper logsMapper, LogsService service, EmailService emailService, Map workInProgressMap) { + private final Executor executor; + + public LogsListener(ObjectMapper mapper, UtilsMapper logsMapper, LogsService service, EmailService emailService, Map workInProgressMap, Executor executor) { this.mapper = mapper; this.logsMapper = logsMapper; this.service = service; this.emailService = emailService; this.workInProgressMap = workInProgressMap; + this.executor = executor; } @@ -60,46 +69,38 @@ public void listenInfoKbart2KafkaAndErrorKbart2Kafka(ConsumerRecord 1) ? key[1] : "-1"))); String packageName = key[0]; - if (!this.workInProgressMap.containsKey(packageName)) { - //nouveau fichier trouvé dans le topic, on initialise les variables partagées - log.debug("Nouveau package identifié : " + packageName); - workInProgressMap.put(packageName, new WorkInProgress()); - } - workInProgressMap.get(packageName).addMessage(dto); - if (!packageName.contains("ctx:package") && !packageName.contains("_FORCE")) { + if (!packageName.equals("${ctx:package}")) { + if (!this.workInProgressMap.containsKey(packageName)) { + //nouveau fichier trouvé dans le topic, on initialise les variables partagées + log.debug("Nouveau package identifié : " + packageName); + workInProgressMap.put(packageName, new WorkInProgress()); + } + LogKbart logKbart = logsMapper.map(dto, LogKbart.class); + logKbart.setPackageName(packageName); + logKbart.setTimestamp(new Date(message.timestamp())); + workInProgressMap.get(packageName).addMessage(logKbart); + if ((dto.getMessage().contains("Traitement terminé pour fichier " + packageName)) || (dto.getMessage().contains("Traitement refusé du fichier " + packageName))) { - log.debug("Commit les datas pour fichier " + packageName); - Integer nbLine = Integer.parseInt(((key.length > 1) ? key[1] : "-1")); - Integer nbRun = commitDatas(message.timestamp(), packageName, nbLine); - createFileBad(packageName, nbRun); + saveDatas(workInProgressMap.get(packageName).getMessages()); + if (!packageName.contains("_FORCE")) { + createFileBad(packageName); + } workInProgressMap.remove(packageName); } } } - private Integer commitDatas(long timeStamp, String packageName, Integer nbLine) { - long startTime = System.currentTimeMillis(); - log.debug("Debut Commit datas pour fichier " + packageName); - List logskbart = logsMapper.mapList(workInProgressMap.get(packageName).getMessages(), LogKbart.class); - int nbRun = service.getLastNbRun(packageName) + 1; - log.debug("NbRun: " + nbRun); - saveDatas(timeStamp, packageName, nbLine, logskbart, nbRun); - log.debug("datas saved pour fichier " + packageName); - long endTime = System.currentTimeMillis(); - double executionTime = (double) (endTime - startTime) / 1000; - log.debug("Execution time: " + executionTime); - return nbRun; - } - - private void saveDatas(long timeStamp, String packageName, Integer nbLine, List logskbart, int nbRun) { - logskbart.forEach(logKbart -> { - logKbart.setNbRun(nbRun); - logKbart.setTimestamp(new Date(timeStamp)); - logKbart.setPackageName(packageName); - logKbart.setNbLine(nbLine); - }); - service.saveAll(logskbart); + private void saveDatas(List logskbart) { + //découpage de la liste en paquets de maxPacketSize pour sauvegarde dans ES pour éviter le timeout ou une erreur ES + IntStream.range(0, (logskbart.size() + maxPacketSize - 1) / maxPacketSize) + .mapToObj(i -> logskbart.subList(i * maxPacketSize, Math.min((i + 1) * maxPacketSize, logskbart.size()))) + .toList().forEach(logskbartList -> executor.execute(() -> { + log.debug("Saving logskbart : {}", logskbartList.size()); + service.saveAll(logskbartList); + })); + log.debug("Sortie de la sauvegarde"); } public void deleteOldLocalTempLog() throws IOException { @@ -121,8 +122,11 @@ public void deleteOldLocalTempLog() throws IOException { } } - private void createFileBad(String filename, Integer nbRun) throws IOException { - List logKbartList = service.getErrorLogKbartByPackageAndNbRun(filename, nbRun); + private void createFileBad(String filename) throws IOException { + log.debug("Entrée dans createFileBad : {}", filename); + List logskbartList = workInProgressMap.get(filename).getMessages().stream().filter(message -> message.getLevel().equals("ERROR")).sorted().toList(); + log.debug("Taille liste : " + logskbartList.size()); + //List logKbartList = service.getErrorLogKbartByPackageAndNbRun(filename, nbRun); Path tempPath = Path.of("tempLogLocal"); if (!Files.exists(tempPath)) { Files.createDirectory(tempPath); @@ -131,7 +135,7 @@ private void createFileBad(String filename, Integer nbRun) throws IOException { // vérifie la présence de fichiers obsolètes dans le répertoire tempLogLocal et les supprime le cas échéant deleteOldLocalTempLog(); - logKbartList.forEach(logKbart -> { + logskbartList.forEach(logKbart -> { try { if (Files.exists(pathOfBadLocal)) { // Inscrit la ligne dedans @@ -140,7 +144,7 @@ private void createFileBad(String filename, Integer nbRun) throws IOException { // Créer le fichier et inscrit la ligne dedans Files.createFile(pathOfBadLocal); // Créer la ligne d'en-tête - Files.write(pathOfBadLocal, ("LINE\tMESSAGE\t(" + nbRun + ")" + System.lineSeparator()).getBytes(), StandardOpenOption.APPEND); + Files.write(pathOfBadLocal, ("LINE\tMESSAGE\t" + System.lineSeparator()).getBytes(), StandardOpenOption.APPEND); // Inscrit les informations sur la ligne Files.write(pathOfBadLocal, (logKbart.getNbLine() + "\t" + logKbart.getMessage() + System.lineSeparator()).getBytes(), StandardOpenOption.APPEND); log.info("Fichier temporaire créé."); diff --git a/src/main/java/fr/abes/logskbart/kafka/WorkInProgress.java b/src/main/java/fr/abes/logskbart/kafka/WorkInProgress.java index 226ca31..5fa3238 100644 --- a/src/main/java/fr/abes/logskbart/kafka/WorkInProgress.java +++ b/src/main/java/fr/abes/logskbart/kafka/WorkInProgress.java @@ -15,7 +15,7 @@ @Setter @Slf4j public class WorkInProgress { - private List messages; + private List messages; private Timestamp timestamp; @@ -23,7 +23,7 @@ public WorkInProgress() { this.messages = Collections.synchronizedList(new ArrayList<>()); } - public void addMessage(LogKbartDto message) { + public void addMessage(LogKbart message) { this.messages.add(message); } diff --git a/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java b/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java index 1b35e98..1979bab 100644 --- a/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java +++ b/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java @@ -11,7 +11,5 @@ public interface LogKbartRepository extends ElasticsearchRepository { List findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(String filename, Date debut, Date fin); - List findAllByPackageNameAndNbRunAndLevelOrderByNbLineAscTimestampAsc(String filename, Integer nbRun, String level); - List findByPackageNameOrderByNbRunDesc(String filename); } diff --git a/src/main/java/fr/abes/logskbart/service/LogsService.java b/src/main/java/fr/abes/logskbart/service/LogsService.java index e06ad17..e9b326c 100644 --- a/src/main/java/fr/abes/logskbart/service/LogsService.java +++ b/src/main/java/fr/abes/logskbart/service/LogsService.java @@ -2,21 +2,18 @@ import fr.abes.logskbart.entity.LogKbart; import fr.abes.logskbart.repository.LogKbartRepository; -import fr.abes.logskbart.utils.Level; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import java.util.Calendar; import java.util.Date; import java.util.List; -import java.util.Optional; @Service @Slf4j public class LogsService { private final LogKbartRepository repository; - public LogsService(LogKbartRepository repository) { this.repository = repository; } @@ -32,19 +29,10 @@ public List getLogKbartForPackage(String packageName, Date date) { public void saveAll(List logKbarts) { repository.saveAll(logKbarts); + log.debug("Save done !"); } - public Integer getLastNbRun(String packageName) { - List logskbart = repository.findByPackageNameOrderByNbRunDesc(packageName); - Optional logKbart = logskbart.stream().findFirst(); - if (logKbart.isPresent()) { - Integer nbRun = logKbart.get().getNbRun(); - return (nbRun != null) ? nbRun : 0; - } - return 0; - } - - public List getErrorLogKbartByPackageAndNbRun(String packageName, Integer nbRun) { - return repository.findAllByPackageNameAndNbRunAndLevelOrderByNbLineAscTimestampAsc(packageName,nbRun, String.valueOf(Level.ERROR)); + public void save(LogKbart logKbart) { + repository.save(logKbart); } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 6078f36..1764bd9 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -4,14 +4,12 @@ # Configuration du projet (depuis pom.xml) spring.profiles.active=@spring.profiles.active@ -application.name=@project.artifactId@ -application.version=@project.version@ -application.basedir=@webBaseDir@ # Configuration du serveur Http server.port=8082 logging.config=classpath:log4j2.xml +elasticsearch.max-packet-size=7500 # Topic Kafka topic.name.source.error=bacon.logs.toload From 8696ad537f0bf922653cacf56ae3a3155dba646e Mon Sep 17 00:00:00 2001 From: pierre-maraval Date: Wed, 23 Oct 2024 07:56:14 +0200 Subject: [PATCH 6/8] FIX : Ajout variable nbThread dans application.properties --- src/main/resources/application-dev.properties | 2 +- src/main/resources/application-prod.properties | 2 +- src/main/resources/application-test.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 35204bf..98a0e40 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -1,6 +1,6 @@ # Consumer properties abes.kafka.bootstrap-servers= -abes.kafka.concurrency.nbThread= +abes.nbThread= # Properties defined from .env on server #ignore resolution error spring.elasticsearch.uris= diff --git a/src/main/resources/application-prod.properties b/src/main/resources/application-prod.properties index a511fcf..6457cde 100644 --- a/src/main/resources/application-prod.properties +++ b/src/main/resources/application-prod.properties @@ -1,6 +1,6 @@ # Consumer properties abes.kafka.bootstrap-servers= -abes.kafka.concurrency.nbThread= +abes.nbThread= # Base Postgres # Properties defined from .env on server #ignore resolution error diff --git a/src/main/resources/application-test.properties b/src/main/resources/application-test.properties index 45482d8..d7e7bf9 100644 --- a/src/main/resources/application-test.properties +++ b/src/main/resources/application-test.properties @@ -1,6 +1,6 @@ # Consumer properties abes.kafka.bootstrap-servers= -abes.kafka.concurrency.nbThread= +abes.nbThread= # Properties defined from .env on server #ignore resolution error spring.elasticsearch.uris= From 7cab97ea474feb0be30273d365e72f8dca48b3f9 Mon Sep 17 00:00:00 2001 From: pierre-maraval Date: Wed, 23 Oct 2024 15:28:37 +0200 Subject: [PATCH 7/8] =?UTF-8?q?FIX=20:=20Modification=20index=20pour=20opt?= =?UTF-8?q?imisation=20recherches=20Suppression=20liste=20dans=20retour=20?= =?UTF-8?q?controller=20Ajout=20insertion=20dans=20un=20fichier=20renvoy?= =?UTF-8?q?=C3=A9=20par=20le=20controller=20pour=20r=C3=A9cup=C3=A9rer=20l?= =?UTF-8?q?es=20logs=20d'un=20package=20Mise=20=C3=A0=20jour=20versions=20?= =?UTF-8?q?librairies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/docker-entrypoint.sh | 10 ++++- pom.xml | 20 +++------ .../logskbart/controller/LogsController.java | 40 ++++++++++++----- .../fr/abes/logskbart/entity/LogKbart.java | 2 - .../exception/EmptyFileException.java | 7 +++ .../exception/ExceptionControllerHandler.java | 4 ++ .../repository/LogKbartRepository.java | 7 +-- .../abes/logskbart/service/LogsService.java | 45 +++++++++++++++++-- 8 files changed, 100 insertions(+), 35 deletions(-) create mode 100644 src/main/java/fr/abes/logskbart/exception/EmptyFileException.java diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 6a607fd..8d946ea 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -5,7 +5,7 @@ export SPRING_ELASTICSEARCH_URIS=${SPRING_ELASTICSEARCH_URIS:='http://localhost: curl -X PUT "$SPRING_ELASTICSEARCH_URIS/logkbart" -H 'Content-Type: application/json' -d' { "settings": { - "number_of_shards": 5, + "number_of_shards": 1, "number_of_replicas": 0 }, "mappings": { @@ -14,7 +14,13 @@ curl -X PUT "$SPRING_ELASTICSEARCH_URIS/logkbart" -H 'Content-Type: application/ "type": "text" }, "PACKAGE_NAME": { - "type": "text" + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } }, "TIMESTAMP": { "type": "date" diff --git a/pom.xml b/pom.xml index 31b7e5d..edaef9b 100644 --- a/pom.xml +++ b/pom.xml @@ -87,6 +87,12 @@ org.springframework.boot spring-boot-starter-data-elasticsearch + + + org.apache.httpcomponents + httpclient + + @@ -105,12 +111,11 @@ - org.modelmapper modelmapper - 2.3.5 + 3.1.1 @@ -139,17 +144,6 @@ 2.3.0 - - - org.springframework.boot - spring-boot-starter-json - - - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - 2.17.0 - - org.springframework.boot diff --git a/src/main/java/fr/abes/logskbart/controller/LogsController.java b/src/main/java/fr/abes/logskbart/controller/LogsController.java index 8a353d5..4b86756 100644 --- a/src/main/java/fr/abes/logskbart/controller/LogsController.java +++ b/src/main/java/fr/abes/logskbart/controller/LogsController.java @@ -2,6 +2,8 @@ import fr.abes.logskbart.dto.LigneLogDto; import fr.abes.logskbart.dto.LogWebDto; +import fr.abes.logskbart.entity.LogKbart; +import fr.abes.logskbart.exception.EmptyFileException; import fr.abes.logskbart.service.LogsService; import fr.abes.logskbart.utils.UtilsMapper; import io.swagger.v3.oas.annotations.Operation; @@ -9,8 +11,16 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import org.springframework.core.io.InputStreamResource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; @@ -23,29 +33,37 @@ public class LogsController { private final LogsService service; - private final UtilsMapper mapper; - public LogsController(LogsService service, UtilsMapper mapper) { + public LogsController(LogsService service) { this.service = service; - this.mapper = mapper; } @Operation( summary = "Transfer kafka to PostgreSQL DB", description = "Retrieves kbart load logs from a Kafka bus and stores them in a DB for later availability", responses = { - @ApiResponse( responseCode = "200", description = "The request was successful.", content = { @Content(schema = @Schema(implementation = LogWebDto.class), mediaType = "application/json") } ), - @ApiResponse( responseCode = "400", description = "An element of the query is badly formulated.", content = { @Content(schema = @Schema()) } ), - @ApiResponse( responseCode = "500", description = "An internal server error interrupted processing.", content = { @Content(schema = @Schema()) } ), + @ApiResponse(responseCode = "200", description = "The request was successful.", content = {@Content(schema = @Schema(implementation = LogWebDto.class), mediaType = "application/json")}), + @ApiResponse(responseCode = "400", description = "An element of the query is badly formulated.", content = {@Content(schema = @Schema())}), + @ApiResponse(responseCode = "500", description = "An internal server error interrupted processing.", content = {@Content(schema = @Schema())}), } ) @GetMapping("/logs/{filename}/{date}") - public LogWebDto getLogsFromPackageAndDate(@PathVariable String filename, @PathVariable String date) throws ParseException { + public ResponseEntity getLogsFromPackageAndDate(@PathVariable String filename, @PathVariable String date) throws ParseException, IOException, EmptyFileException { SimpleDateFormat format = new SimpleDateFormat("ddMMyyyy"); Date dateAnalyse = format.parse(date); - LogWebDto logWebDto = new LogWebDto(filename, date); - List lignes = mapper.mapList(service.getLogKbartForPackage(filename, dateAnalyse), LigneLogDto.class); - logWebDto.addLignes(lignes); - return logWebDto; + File fichier = service.getLogKbartForPackage(filename, dateAnalyse); + FileInputStream fs = new FileInputStream(fichier); + // Définir les en-têtes pour la réponse HTTP + HttpHeaders headers = new HttpHeaders(); + headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + fichier.getName()); + headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE); + + // Retourner la réponse avec le fichier à télécharger + return ResponseEntity.ok() + .headers(headers) + .contentLength(fichier.length()) + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(new InputStreamResource(fs)); + } } diff --git a/src/main/java/fr/abes/logskbart/entity/LogKbart.java b/src/main/java/fr/abes/logskbart/entity/LogKbart.java index a5ed3ff..5266e35 100644 --- a/src/main/java/fr/abes/logskbart/entity/LogKbart.java +++ b/src/main/java/fr/abes/logskbart/entity/LogKbart.java @@ -1,6 +1,5 @@ package fr.abes.logskbart.entity; -import fr.abes.logskbart.utils.Level; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.data.annotation.Id; @@ -8,7 +7,6 @@ import org.springframework.data.elasticsearch.annotations.Field; import java.io.Serializable; -import java.util.Comparator; import java.util.Date; import java.util.Objects; diff --git a/src/main/java/fr/abes/logskbart/exception/EmptyFileException.java b/src/main/java/fr/abes/logskbart/exception/EmptyFileException.java new file mode 100644 index 0000000..84c47db --- /dev/null +++ b/src/main/java/fr/abes/logskbart/exception/EmptyFileException.java @@ -0,0 +1,7 @@ +package fr.abes.logskbart.exception; + +public class EmptyFileException extends Throwable { + public EmptyFileException(String message) { + super(message); + } +} diff --git a/src/main/java/fr/abes/logskbart/exception/ExceptionControllerHandler.java b/src/main/java/fr/abes/logskbart/exception/ExceptionControllerHandler.java index 157be7d..b42312c 100644 --- a/src/main/java/fr/abes/logskbart/exception/ExceptionControllerHandler.java +++ b/src/main/java/fr/abes/logskbart/exception/ExceptionControllerHandler.java @@ -19,6 +19,10 @@ private ResponseEntity buildResponseEntity(ApiReturnError apiReturnError return new ResponseEntity<>(apiReturnError, apiReturnError.getStatus()); } + @ExceptionHandler(EmptyFileException.class) + protected ResponseEntity handleEmptyFileException(EmptyFileException e) { + return buildResponseEntity(new ApiReturnError(HttpStatus.NOT_FOUND, e.getMessage())); + } /** * Erreur dans la validité des paramètres de la requête * diff --git a/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java b/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java index 1979bab..72dde8f 100644 --- a/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java +++ b/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java @@ -1,15 +1,16 @@ package fr.abes.logskbart.repository; import fr.abes.logskbart.entity.LogKbart; +import org.springframework.data.domain.*; import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; import org.springframework.stereotype.Repository; import java.util.Date; -import java.util.List; +import java.util.stream.Stream; @Repository public interface LogKbartRepository extends ElasticsearchRepository { - List findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(String filename, Date debut, Date fin); - + long countByPackageNameAndTimestampBetween(String packageName, Date debut, Date fin); + Stream findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(String filename, Date debut, Date fin); } diff --git a/src/main/java/fr/abes/logskbart/service/LogsService.java b/src/main/java/fr/abes/logskbart/service/LogsService.java index e9b326c..ed7affa 100644 --- a/src/main/java/fr/abes/logskbart/service/LogsService.java +++ b/src/main/java/fr/abes/logskbart/service/LogsService.java @@ -1,30 +1,67 @@ package fr.abes.logskbart.service; import fr.abes.logskbart.entity.LogKbart; +import fr.abes.logskbart.exception.EmptyFileException; import fr.abes.logskbart.repository.LogKbartRepository; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Calendar; import java.util.Date; import java.util.List; +import java.util.stream.Stream; @Service @Slf4j public class LogsService { + private static final Integer MAXSIZE = 5000; private final LogKbartRepository repository; - public LogsService(LogKbartRepository repository) { - this.repository = repository; + public LogsService(LogKbartRepository logKbartRepository) { + this.repository = logKbartRepository; } - public List getLogKbartForPackage(String packageName, Date date) { + @Transactional(readOnly = true) + public File getLogKbartForPackage(String packageName, Date date) throws IOException, EmptyFileException { Calendar dateChargement = Calendar.getInstance(); dateChargement.setTime(date); Calendar dateFin = (Calendar) dateChargement.clone(); dateFin.add(Calendar.DAY_OF_MONTH, 1); log.debug("packageName {}, Date début {}, Date fin {}", packageName, dateChargement.getTime(), dateFin.getTime()); - return repository.findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(packageName, dateChargement.getTime(), dateFin.getTime()); + if (repository.countByPackageNameAndTimestampBetween(packageName, dateChargement.getTime(), dateFin.getTime()) == 0) + throw new EmptyFileException("Aucun log pour le fichier " + packageName); + Path tempPath = Path.of("tempLogLocal"); + if (!Files.exists(tempPath)) { + Files.createDirectory(tempPath); + } + Path pathOfLocal = Path.of("tempLogLocal" + File.separator + packageName.replace(".tsv", ".log")); + Files.deleteIfExists(pathOfLocal); + try (Stream logKbarts = repository.findAllByPackageNameAndTimestampBetweenOrderByNbLineAscTimestampAsc(packageName, dateChargement.getTime(), dateFin.getTime())) { + logKbarts.forEach(logKbart -> { + String message = (logKbart.getNbLine() != -1) ? logKbart.getNbLine() + " : " : ""; + message += logKbart.getMessage() + System.lineSeparator(); + try { + if (Files.exists(pathOfLocal)) { + Files.write(pathOfLocal, message.getBytes(), StandardOpenOption.APPEND); + } else { + // Créer le fichier et inscrit la ligne dedans + Files.createFile(pathOfLocal); + // Inscrit les informations sur la ligne + Files.write(pathOfLocal, message.getBytes(), StandardOpenOption.APPEND); + log.debug("Fichier temporaire créé."); + } + }catch (IOException e) { + log.error("Impossible d'écrire dans le fichier local", e); + } + }); + } + return pathOfLocal.toFile(); } public void saveAll(List logKbarts) { From b962a7b8e5fa362ebf89f3a33cc25c4a0cfa14b3 Mon Sep 17 00:00:00 2001 From: pierre-maraval Date: Thu, 24 Oct 2024 09:13:03 +0200 Subject: [PATCH 8/8] =?UTF-8?q?FIX=20:=20Suppression=20code=20obsol=C3=A8t?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fr/abes/logskbart/repository/LogKbartRepository.java | 1 - src/main/java/fr/abes/logskbart/service/LogsService.java | 5 ----- 2 files changed, 6 deletions(-) diff --git a/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java b/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java index 72dde8f..47dbad4 100644 --- a/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java +++ b/src/main/java/fr/abes/logskbart/repository/LogKbartRepository.java @@ -1,7 +1,6 @@ package fr.abes.logskbart.repository; import fr.abes.logskbart.entity.LogKbart; -import org.springframework.data.domain.*; import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; import org.springframework.stereotype.Repository; diff --git a/src/main/java/fr/abes/logskbart/service/LogsService.java b/src/main/java/fr/abes/logskbart/service/LogsService.java index ed7affa..ab223ed 100644 --- a/src/main/java/fr/abes/logskbart/service/LogsService.java +++ b/src/main/java/fr/abes/logskbart/service/LogsService.java @@ -20,7 +20,6 @@ @Service @Slf4j public class LogsService { - private static final Integer MAXSIZE = 5000; private final LogKbartRepository repository; public LogsService(LogKbartRepository logKbartRepository) { @@ -68,8 +67,4 @@ public void saveAll(List logKbarts) { repository.saveAll(logKbarts); log.debug("Save done !"); } - - public void save(LogKbart logKbart) { - repository.save(logKbart); - } }