From 3066d7c1454b6b6c69870c5055ff124ecff73380 Mon Sep 17 00:00:00 2001 From: hyunseok Date: Mon, 26 Oct 2020 09:44:50 +0900 Subject: [PATCH] =?UTF-8?q?elasticsearch=20=20=EB=A1=9C=EA=B1=B0=20?= =?UTF-8?q?=EC=82=AD=EC=A0=9C.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 2 - .../AbstractCustomElasticsearchAppender.java | 142 ------------- .../AbstractCustomElasticsearchPublisher.java | 199 ------------------ .../logger/CustomElasticsearchAggregator.java | 76 ------- .../logger/CustomElasticsearchAppender.java | 56 ----- .../logger/CustomElasticsearchPublisher.java | 76 ------- src/main/resources/logback-spring.xml | 45 +--- 7 files changed, 1 insertion(+), 595 deletions(-) delete mode 100644 src/main/java/com/msws/shareplates/framework/logger/AbstractCustomElasticsearchAppender.java delete mode 100644 src/main/java/com/msws/shareplates/framework/logger/AbstractCustomElasticsearchPublisher.java delete mode 100644 src/main/java/com/msws/shareplates/framework/logger/CustomElasticsearchAggregator.java delete mode 100644 src/main/java/com/msws/shareplates/framework/logger/CustomElasticsearchAppender.java delete mode 100644 src/main/java/com/msws/shareplates/framework/logger/CustomElasticsearchPublisher.java diff --git a/build.gradle b/build.gradle index caa1a76..2071622 100644 --- a/build.gradle +++ b/build.gradle @@ -40,8 +40,6 @@ dependencies { implementation 'com.github.miwurster:spring-data-influxdb:1.8' implementation 'org.bgee.log4jdbc-log4j2:log4jdbc-log4j2-jdbc4:1.16' - - implementation 'com.internetitem:logback-elasticsearch-appender:1.6' // https://mvnrepository.com/artifact/org.hibernate/hibernate-validator compile group: 'org.hibernate', name: 'hibernate-validator', version: '6.1.4.Final' diff --git a/src/main/java/com/msws/shareplates/framework/logger/AbstractCustomElasticsearchAppender.java b/src/main/java/com/msws/shareplates/framework/logger/AbstractCustomElasticsearchAppender.java deleted file mode 100644 index a428790..0000000 --- a/src/main/java/com/msws/shareplates/framework/logger/AbstractCustomElasticsearchAppender.java +++ /dev/null @@ -1,142 +0,0 @@ -package com.msws.shareplates.framework.logger; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; - -import com.internetitem.logback.elasticsearch.config.Authentication; -import com.internetitem.logback.elasticsearch.config.ElasticsearchProperties; -import com.internetitem.logback.elasticsearch.config.HttpRequestHeaders; -import com.internetitem.logback.elasticsearch.config.Settings; -import com.internetitem.logback.elasticsearch.util.ErrorReporter; - -import ch.qos.logback.core.UnsynchronizedAppenderBase; - -public abstract class AbstractCustomElasticsearchAppender extends UnsynchronizedAppenderBase { - - protected Settings settings; - protected ElasticsearchProperties elasticsearchProperties; - protected AbstractCustomElasticsearchPublisher publisher; - protected ErrorReporter errorReporter; - protected HttpRequestHeaders headers; - - public AbstractCustomElasticsearchAppender() { - this.settings = new Settings(); - this.headers = new HttpRequestHeaders(); - } - - public AbstractCustomElasticsearchAppender(Settings settings) { - this.settings = settings; - } - - @Override - public void start() { - super.start(); - this.errorReporter = getErrorReporter(); - try { - this.publisher = buildElasticsearchPublisher(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - protected void publishEvent(T eventObject) { - publisher.addEvent(eventObject); - } - - //VisibleForTesting - protected ErrorReporter getErrorReporter() { - return new ErrorReporter(settings, getContext()); - } - - //VisibleForTesting - protected abstract AbstractCustomElasticsearchPublisher buildElasticsearchPublisher() throws IOException; - - @Override - public void stop() { - super.stop(); - } - - @Override - protected void append(T eventObject) { - appendInternal(eventObject); - } - - protected abstract void appendInternal(T eventObject); - - public void setProperties(ElasticsearchProperties elasticsearchProperties) { - this.elasticsearchProperties = elasticsearchProperties; - } - - public void setSleepTime(int sleepTime) { - settings.setSleepTime(sleepTime); - } - - public void setMaxRetries(int maxRetries) { - settings.setMaxRetries(maxRetries); - } - - public void setConnectTimeout(int connectTimeout) { - settings.setConnectTimeout(connectTimeout); - } - - public void setReadTimeout(int readTimeout) { - settings.setReadTimeout(readTimeout); - } - - public void setIncludeCallerData(boolean includeCallerData) { - settings.setIncludeCallerData(includeCallerData); - } - - public void setErrorsToStderr(boolean errorsToStderr) { - settings.setErrorsToStderr(errorsToStderr); - } - - public void setLogsToStderr(boolean logsToStderr) { - settings.setLogsToStderr(logsToStderr); - } - - public void setMaxQueueSize(int maxQueueSize) { - settings.setMaxQueueSize(maxQueueSize); - } - - public void setIndex(String index) { - settings.setIndex(index); - } - - public void setType(String type) { - settings.setType(type); - } - - public void setUrl(String url) throws MalformedURLException { - settings.setUrl(new URL(url)); - } - - public void setLoggerName(String logger) { - settings.setLoggerName(logger); - } - - public void setErrorLoggerName(String logger) { - settings.setErrorLoggerName(logger); - } - - public void setHeaders(HttpRequestHeaders httpRequestHeaders) { - this.headers = httpRequestHeaders; - } - - public void setRawJsonMessage(boolean rawJsonMessage) { - settings.setRawJsonMessage(rawJsonMessage); - } - - public void setIncludeMdc(boolean includeMdc) { - settings.setIncludeMdc(includeMdc); - } - - public void setAuthentication(Authentication auth) { - settings.setAuthentication(auth); - } - - public void setMaxMessageSize(int maxMessageSize) { - settings.setMaxMessageSize(maxMessageSize); - } -} \ No newline at end of file diff --git a/src/main/java/com/msws/shareplates/framework/logger/AbstractCustomElasticsearchPublisher.java b/src/main/java/com/msws/shareplates/framework/logger/AbstractCustomElasticsearchPublisher.java deleted file mode 100644 index b768a08..0000000 --- a/src/main/java/com/msws/shareplates/framework/logger/AbstractCustomElasticsearchPublisher.java +++ /dev/null @@ -1,199 +0,0 @@ -package com.msws.shareplates.framework.logger; - -import java.io.IOException; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import com.internetitem.logback.elasticsearch.config.ElasticsearchProperties; -import com.internetitem.logback.elasticsearch.config.HttpRequestHeaders; -import com.internetitem.logback.elasticsearch.config.Property; -import com.internetitem.logback.elasticsearch.config.Settings; -import com.internetitem.logback.elasticsearch.util.AbstractPropertyAndEncoder; -import com.internetitem.logback.elasticsearch.util.ErrorReporter; -import com.internetitem.logback.elasticsearch.writer.ElasticsearchWriter; -import com.internetitem.logback.elasticsearch.writer.LoggerWriter; -import com.internetitem.logback.elasticsearch.writer.StdErrWriter; - -import ch.qos.logback.core.Context; - -public abstract class AbstractCustomElasticsearchPublisher implements Runnable { - - private static final AtomicInteger THREAD_COUNTER = new AtomicInteger(1); - private static final ThreadLocal DATE_FORMAT = new ThreadLocal () { - @Override - protected DateFormat initialValue() { - return new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - } - }; - - public static final String THREAD_NAME_PREFIX = "es-writer-"; - - - private volatile List events; - private CustomElasticsearchAggregator outputAggregator; - private List> propertyList; - - private AbstractPropertyAndEncoder indexPattern; - private JsonFactory jf; - private JsonGenerator jsonGenerator; - - private ErrorReporter errorReporter; - protected Settings settings; - - private final Object lock; - - private volatile boolean working; - - public AbstractPropertyAndEncoder getIndexPattern(){ - return indexPattern; - } - - public AbstractCustomElasticsearchPublisher(Context context, ErrorReporter errorReporter, Settings settings, ElasticsearchProperties properties, HttpRequestHeaders headers) throws IOException { - this.errorReporter = errorReporter; - this.events = new ArrayList(); - this.lock = new Object(); - this.settings = settings; - - this.outputAggregator = configureOutputAggregator(settings, errorReporter, headers); - - this.jf = new JsonFactory(); - this.jf.setRootValueSeparator(null); - this.jsonGenerator = jf.createGenerator(outputAggregator); - - this.indexPattern = buildPropertyAndEncoder(context, new Property("", settings.getIndex(), false)); - this.propertyList = generatePropertyList(context, properties); - - } - - private static CustomElasticsearchAggregator configureOutputAggregator(Settings settings, ErrorReporter errorReporter, HttpRequestHeaders httpRequestHeaders) { - CustomElasticsearchAggregator spigot = new CustomElasticsearchAggregator(settings, errorReporter); - - if (settings.isLogsToStderr()) { - spigot.addWriter(new StdErrWriter()); - } - - if (settings.getLoggerName() != null) { - spigot.addWriter(new LoggerWriter(settings.getLoggerName())); - } - - if (settings.getUrl() != null) { - spigot.addWriter(new ElasticsearchWriter(errorReporter, settings, httpRequestHeaders)); - } - - return spigot; - } - - private List> generatePropertyList(Context context, ElasticsearchProperties properties) { - List> list = new ArrayList>(); - if (properties != null) { - for (Property property : properties.getProperties()) { - list.add(buildPropertyAndEncoder(context, property)); - } - } - return list; - } - - protected abstract AbstractPropertyAndEncoder buildPropertyAndEncoder(Context context, Property property); - - public void addEvent(T event) { - if (!outputAggregator.hasOutputs()) { - return; - } - - synchronized (lock) { - events.add(event); - if (!working) { - working = true; - Thread thread = new Thread(this, THREAD_NAME_PREFIX + THREAD_COUNTER.getAndIncrement()); - thread.start(); - } - } - } - - public void run() { - int currentTry = 1; - int maxRetries = settings.getMaxRetries(); - while (true) { - try { - Thread.sleep(settings.getSleepTime()); - - List eventsCopy = null; - synchronized (lock) { - if (!events.isEmpty()) { - eventsCopy = events; - events = new ArrayList(); - currentTry = 1; - } - - if (eventsCopy == null) { - if (!outputAggregator.hasPendingData()) { - // all done - working = false; - return; - } else { - // Nothing new, must be a retry - if (currentTry > maxRetries) { - // Oh well, better luck next time - working = false; - return; - } - } - } - } - - if (eventsCopy != null) { - serializeEvents(jsonGenerator, eventsCopy, propertyList); - } - - if (!outputAggregator.sendData()) { - currentTry++; - } - } catch (Exception e) { - errorReporter.logError("Internal error handling log data: " + e.getMessage(), e); - currentTry++; - } - } - } - - - private void serializeEvents(JsonGenerator gen, List eventsCopy, List> propertyList) throws IOException { - for (T event : eventsCopy) { - serializeIndexString(gen, event); - gen.writeRaw('\n'); - serializeEvent(gen, event, propertyList); - gen.writeRaw('\n'); - } - gen.flush(); - } - - private void serializeEvent(JsonGenerator gen, T event, List> propertyList) throws IOException { - gen.writeStartObject(); - - serializeCommonFields(gen, event); - - for (AbstractPropertyAndEncoder pae : propertyList) { - String value = pae.encode(event); - if(pae.allowEmpty() || (value != null && !value.isEmpty())) { - gen.writeObjectField(pae.getName(), value); - } - - } - - gen.writeEndObject(); - } - - protected abstract void serializeCommonFields(JsonGenerator gen, T event) throws IOException; - - protected abstract void serializeIndexString(JsonGenerator gen, T event) throws IOException; - - protected static String getTimestamp(long timestamp) { - return DATE_FORMAT.get().format(new Date(timestamp)); - } - -} diff --git a/src/main/java/com/msws/shareplates/framework/logger/CustomElasticsearchAggregator.java b/src/main/java/com/msws/shareplates/framework/logger/CustomElasticsearchAggregator.java deleted file mode 100644 index 18a07b8..0000000 --- a/src/main/java/com/msws/shareplates/framework/logger/CustomElasticsearchAggregator.java +++ /dev/null @@ -1,76 +0,0 @@ -package com.msws.shareplates.framework.logger; - -import java.io.IOException; -import java.io.Writer; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import com.internetitem.logback.elasticsearch.config.Settings; -import com.internetitem.logback.elasticsearch.util.ErrorReporter; -import com.internetitem.logback.elasticsearch.writer.SafeWriter; - -public class CustomElasticsearchAggregator extends Writer { - - private Settings settings; - private ErrorReporter errorReporter; - private List writers; - - public CustomElasticsearchAggregator(Settings settings, ErrorReporter errorReporter) { - this.writers = new ArrayList(); - this.settings = settings; - this.errorReporter = errorReporter; - } - - public void addWriter(SafeWriter writer) { - writers.add(writer); - } - - @Override - public void write(char[] cbuf, int off, int len) throws IOException { - for (SafeWriter writer : writers) { - writer.write(cbuf, off, len); - } - } - - public boolean hasPendingData() { - for (SafeWriter writer : writers) { - if (writer.hasPendingData()) { - return true; - } - } - return false; - } - - public boolean hasOutputs() { - return !writers.isEmpty(); - } - - public boolean sendData() { - boolean success = true; - for (SafeWriter writer : writers) { - try { - writer.sendData(); - } catch (IOException e) { - success = false; - errorReporter.logWarning("Failed to send events to Elasticsearch: " + e.getMessage()); - if (settings.isErrorsToStderr()) { - System.err.println("[" + new Date().toString() + "] Failed to send events to Elasticsearch: " + e.getMessage()); - } - - } - } - return success; - } - - @Override - public void flush() throws IOException { - // No-op - } - - @Override - public void close() throws IOException { - // No-op - } - -} \ No newline at end of file diff --git a/src/main/java/com/msws/shareplates/framework/logger/CustomElasticsearchAppender.java b/src/main/java/com/msws/shareplates/framework/logger/CustomElasticsearchAppender.java deleted file mode 100644 index 207d852..0000000 --- a/src/main/java/com/msws/shareplates/framework/logger/CustomElasticsearchAppender.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.msws.shareplates.framework.logger; - -import java.io.IOException; - -import com.internetitem.logback.elasticsearch.config.Settings; - -import ch.qos.logback.classic.spi.ILoggingEvent; - -public class CustomElasticsearchAppender extends AbstractCustomElasticsearchAppender { - - private Settings currentSettings; - - public Settings getCurrentSettings() { - return this.currentSettings; - } - - public void setCurrentSettings(Settings currentSettings) { - this.currentSettings = currentSettings; - } - - public CustomElasticsearchAppender() { - } - - public CustomElasticsearchAppender(Settings settings) { - super(settings); - this.currentSettings = settings; - } - - @Override - protected void appendInternal(ILoggingEvent eventObject) { - - String targetLogger = eventObject.getLoggerName(); - - String loggerName = settings.getLoggerName(); - if (loggerName != null && loggerName.equals(targetLogger)) { - return; - } - - String errorLoggerName = settings.getErrorLoggerName(); - if (errorLoggerName != null && errorLoggerName.equals(targetLogger)) { - return; - } - - eventObject.prepareForDeferredProcessing(); - if (settings.isIncludeCallerData()) { - eventObject.getCallerData(); - } - - publishEvent(eventObject); - } - - protected CustomElasticsearchPublisher buildElasticsearchPublisher() throws IOException { - return new CustomElasticsearchPublisher(getContext(), errorReporter, settings, elasticsearchProperties, headers); - } - -} diff --git a/src/main/java/com/msws/shareplates/framework/logger/CustomElasticsearchPublisher.java b/src/main/java/com/msws/shareplates/framework/logger/CustomElasticsearchPublisher.java deleted file mode 100644 index a57d03a..0000000 --- a/src/main/java/com/msws/shareplates/framework/logger/CustomElasticsearchPublisher.java +++ /dev/null @@ -1,76 +0,0 @@ -package com.msws.shareplates.framework.logger; - -import java.io.IOException; -import java.util.Map; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.internetitem.logback.elasticsearch.config.ElasticsearchProperties; -import com.internetitem.logback.elasticsearch.config.HttpRequestHeaders; -import com.internetitem.logback.elasticsearch.config.Property; -import com.internetitem.logback.elasticsearch.config.Settings; -import com.internetitem.logback.elasticsearch.util.AbstractPropertyAndEncoder; -import com.internetitem.logback.elasticsearch.util.ClassicPropertyAndEncoder; -import com.internetitem.logback.elasticsearch.util.ErrorReporter; - -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.Context; - -public class CustomElasticsearchPublisher extends AbstractCustomElasticsearchPublisher { - - public CustomElasticsearchPublisher(Context context, ErrorReporter errorReporter, Settings settings, - ElasticsearchProperties properties, HttpRequestHeaders headers) throws IOException { - super(context, errorReporter, settings, properties, headers); - } - - @Override - protected AbstractPropertyAndEncoder buildPropertyAndEncoder(Context context, Property property) { - return new ClassicPropertyAndEncoder(property, context); - } - - @Override - protected void serializeIndexString(JsonGenerator gen, ILoggingEvent event) throws IOException { - gen.writeStartObject(); - gen.writeObjectFieldStart("index"); - - if(event.getMDCPropertyMap().get("index_name") != null) - gen.writeObjectField("_index", event.getMDCPropertyMap().get("index_name")); - else - gen.writeObjectField("_index", this.getIndexPattern().encode(event)); - - String type = settings.getType(); - if(type != null) { - gen.writeObjectField("_type", type); - } - - gen.writeEndObject(); - gen.writeEndObject(); - - } - - @Override - protected void serializeCommonFields(JsonGenerator gen, ILoggingEvent event) throws IOException { - gen.writeObjectField("@timestamp", getTimestamp(event.getTimeStamp())); - - if (settings.isRawJsonMessage()) { - gen.writeFieldName("message"); - gen.writeRawValue(event.getFormattedMessage()); - } else { - String formattedMessage = event.getFormattedMessage(); - if (settings.getMaxMessageSize() > 0 && formattedMessage.length() > settings.getMaxMessageSize()) { - formattedMessage = formattedMessage.substring(0, settings.getMaxMessageSize()) + ".."; - } - gen.writeObjectField("message", formattedMessage); - } - - - // add MDC data field - if (settings.isIncludeMdc()) { - for (Map.Entry entry : event.getMDCPropertyMap().entrySet()) { - gen.writeObjectField(entry.getKey(), entry.getValue()); - } - } - } - - - -} diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml index 4011c54..f24b663 100644 --- a/src/main/resources/logback-spring.xml +++ b/src/main/resources/logback-spring.xml @@ -6,51 +6,12 @@ - - http://yourserver/_bulk - logs-%date{yyyy-MM-dd} - tester - es-logger - es-error-logger - 30000 - false - false - false - 104857600 - 3 - 30000 - 250 - false - false - 100 - - - - host - ${HOSTNAME} - false - - - logger - ${APP_NAME} - - - -
- Content-Type - application/json -
- - + - - - - @@ -72,10 +33,6 @@ - - - -