Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve LoggingHandler inefficient Issue #154 #614

Draft
wants to merge 1 commit into
base: develop
Choose a base branch
from
+117 −194
Draft

Improve LoggingHandler inefficient Issue #154 #614

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 48 additions & 140 deletions gateleen-logging/src/main/java/org/swisspush/gateleen/logging/LoggingHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,33 +12,33 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.Appender;
import org.apache.logging.log4j.core.appender.RollingFileAppender;
import org.apache.logging.log4j.core.appender.rolling.TimeBasedTriggeringPolicy;
import org.apache.logging.log4j.core.layout.PatternLayout;
import org.slf4j.Logger;
import org.swisspush.gateleen.core.event.EventBusWriter;
import org.swisspush.gateleen.core.http.RequestLoggerFactory;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Pattern;

/**
* Updated LoggingHandler with regex caching
*
* @author https://github.com/mcweba [Marc-Andre Weber]
*/
public class LoggingHandler {

private HttpServerRequest request;
private final HttpServerRequest request;
private MultiMap requestHeaders;
private HttpClientResponse response;
private boolean active = false;

private Buffer requestPayload;
private Buffer responsePayload;
private LoggingResource loggingResource;
private EventBus eventBus;
private LogAppenderRepository logAppenderRepository;
private final LoggingResource loggingResource;
private final LogAppenderRepository logAppenderRepository;

private String currentDestination;

Expand All @@ -49,9 +49,7 @@ public class LoggingHandler {
private static final String DEFAULT_LOGGER = "RequestLog";
private static final String REJECT = "reject";
private static final String DESTINATION = "destination";
private static final String DESCRIPTION = "description";
private static final String META_DATA = "metadata";
private static final String TRANSMISSION = "transmission";

private static final String URL = "url";
private static final String METHOD = "method";
private static final String STATUS_CODE = "statusCode";
Expand All @@ -60,39 +58,43 @@ public class LoggingHandler {
private static final String RESPONSE = "response";
private static final String HEADERS = "headers";
private static final String BODY = "body";
private static final String FILE = "file";
private static final String ADDRESS = "address";
private static final String DEFAULT = "default";

public static final String SKIP_LOGGING_HEADER = "x-skip-request-log";

private Map<String, org.apache.logging.log4j.Logger> loggers = new HashMap<>();
private final Map<String, org.apache.logging.log4j.Logger> loggers = new HashMap<>();
private final Logger log;

// Cache for precompiled regex patterns
private static final Map<String, Pattern> regexCache = new HashMap<>();

private Logger log;
// Helper method to fetch or compile a regex pattern
private static Pattern getOrCompilePattern(String regex) {
return regexCache.computeIfAbsent(regex, Pattern::compile);
}

public LoggingHandler(LoggingResourceManager loggingResourceManager, LogAppenderRepository logAppenderRepository, HttpServerRequest request, EventBus eventBus) {
this.logAppenderRepository = logAppenderRepository;
this.request = request;
this.eventBus = eventBus;
this.loggingResource = loggingResourceManager.getLoggingResource();
this.log = RequestLoggerFactory.getLogger(LoggingHandler.class, request);
((org.apache.logging.log4j.core.Logger) LogManager.getLogger(DEFAULT_LOGGER)).setAdditive(false);
boolean stopValidation = false;

if(request.headers().get(SKIP_LOGGING_HEADER) != null) {
log.info("request will not be logged because of skip log request header");
if (request.headers().get(SKIP_LOGGING_HEADER) != null) {
log.info("Request will not be logged because of skip log request header");
return;
}

boolean stopValidation = false;

for (Map<String, String> payloadFilter : loggingResource.getPayloadFilters()) {
if (active || stopValidation) {
break;
}

// NEMO-5551: Custom sorting. We have to make sure key "URL" comes first in the array.
// Sort "url" key to the head of the list
List<Entry<String, String>> payloadFilterEntrySetList = new ArrayList<>();
for (Entry<String, String> filterEntry : payloadFilter.entrySet()) {
if (filterEntry.getKey().equalsIgnoreCase("url")) {
if (filterEntry.getKey().equalsIgnoreCase(URL)) {
payloadFilterEntrySetList.add(0, filterEntry);
} else {
payloadFilterEntrySetList.add(filterEntry);
Expand All @@ -101,23 +103,17 @@ public LoggingHandler(LoggingResourceManager loggingResourceManager, LogAppender

boolean reject = Boolean.parseBoolean(payloadFilter.get(REJECT));
for (Entry<String, String> filterEntry : payloadFilterEntrySetList) {
if (REJECT.equalsIgnoreCase(filterEntry.getKey())
|| DESTINATION.equalsIgnoreCase(filterEntry.getKey())
|| DESCRIPTION.equalsIgnoreCase(filterEntry.getKey())) {
continue;
}
String key = filterEntry.getKey();
String value = filterEntry.getValue();
Pattern pattern = getOrCompilePattern(value);

FilterResult result = RequestPropertyFilter.filterProperty(request, filterEntry.getKey(), filterEntry.getValue(), reject);
if (result == FilterResult.FILTER) {
if (RequestPropertyFilter.filterProperty(request, key, pattern, reject) == FilterResult.FILTER) {
active = true;
currentDestination = createLoggerAndGetDestination(payloadFilter);
} else if (result == FilterResult.REJECT) {
} else if (RequestPropertyFilter.filterProperty(request, key, pattern, reject) == FilterResult.REJECT) {
active = false;
stopValidation = true;
break;
} else if (result == FilterResult.NO_MATCH) {
active = false;
break;
}
}
}
Expand All @@ -127,134 +123,46 @@ public boolean isActive() {
return this.active;
}

/**
* Returns the destination key for the given filterProperty. If no destination is
* set the default key is used instead. <br />
* A logger for the given destination is created if necessary or reused if
* it already exists.
*
* @param payloadFilter
* @return currentDestination
*/
private String createLoggerAndGetDestination(Map<String, String> payloadFilter) {
// the destination of the active filterProperty
String filterDestination = payloadFilter.get(DESTINATION);

// if not available set to 'default'
if (filterDestination == null) {
log.debug("no filterDestination set");
filterDestination = DEFAULT;
log.debug("No filterDestination set");
filterDestination = DEFAULT_LOGGER;
}

// if the key is found, create a logger for the given file ...
if (loggingResource.getDestinationEntries().containsKey(filterDestination)) {
Map<String, String> destinationOptions = loggingResource.getDestinationEntries().get(filterDestination);

Appender appender = null;
if (destinationOptions.containsKey(FILE)) {
log.debug("found destination entry with type 'file' for: {}", filterDestination);
appender = getFileAppender(filterDestination, destinationOptions.get(FILE));
} else if (destinationOptions.containsKey("address")) {
log.debug("found destination entry with type 'eventBus' for: {}", filterDestination);
appender = getEventBusAppender(filterDestination, destinationOptions);
} else {
log.warn("Unknown typeLocation for destination: {}", filterDestination);
Appender appender = getAppenderForDestination(filterDestination, destinationOptions);
if (appender != null && !loggers.containsKey(filterDestination)) {
org.apache.logging.log4j.Logger logger = LogManager.getLogger("LOG_FILTER_" + payloadFilter.get(URL));
((org.apache.logging.log4j.core.Logger) logger).addAppender(appender);
((org.apache.logging.log4j.core.Logger) logger).setAdditive(false);
loggers.put(filterDestination, logger);
}

if (appender != null) {
if (!loggers.containsKey(filterDestination)) {
org.apache.logging.log4j.Logger filterLogger = LogManager.getLogger("LOG_FILTER_" + payloadFilter.get(URL));
((org.apache.logging.log4j.core.Logger) filterLogger).addAppender(appender);
((org.apache.logging.log4j.core.Logger) filterLogger).setAdditive(false);
loggers.put(filterDestination, filterLogger);
}
} else {
loggers.put(filterDestination, LogManager.getLogger(DEFAULT_LOGGER));
}
}
// ... or use the default logger
else {
if (!filterDestination.equals(DEFAULT)) {
log.warn("no destination entry with name '{}' found, using default logger instead", filterDestination);
}

// use default logger!
} else {
loggers.put(filterDestination, LogManager.getLogger(DEFAULT_LOGGER));
}

return filterDestination;
}

/**
* Returns the eventBus appender matching the given
* filterDestination. If no appender exists for the
* given filterDestination, a new one is created and
* returned.
*
* @param filterDestination
* @param destinationOptions
* @return
*/
private Appender getEventBusAppender(String filterDestination, Map<String, String> destinationOptions) {
if (!logAppenderRepository.hasAppender(filterDestination)) {

/*
* <appender name="requestLogEventBusAppender" class="EventBusAppender">
* <param name="Address" value="event/gateleen-request-log" />
* <layout class="org.apache.log4j.EnhancedPatternLayout">
* <param name="ConversionPattern" value="%m%n" />
* </layout>
* </appender>
*/
EventBusAppender.Builder.setEventBus(eventBus);
EventBusAppender appender = EventBusAppender.newBuilder().setName(filterDestination)
.setAddress(destinationOptions.get(ADDRESS))
.setDeliveryOptionsHeaders(MultiMap.caseInsensitiveMultiMap()
.add(META_DATA, destinationOptions.get(META_DATA)))
.setTransmissionMode(EventBusWriter.TransmissionMode.fromString(destinationOptions.get(TRANSMISSION)))
.setLayout(PatternLayout.createDefaultLayout()).build();
logAppenderRepository.addAppender(filterDestination, appender);
private Appender getAppenderForDestination(String filterDestination, Map<String, String> destinationOptions) {
if (logAppenderRepository.hasAppender(filterDestination)) {
return logAppenderRepository.getAppender(filterDestination);
}
return logAppenderRepository.getAppender(filterDestination);
}

/**
* Returns the file appender matching the given
* filterDestination. If no appender exists for the
* given filterDestination, a new one is created and
* returned.
*
* @param filterDestination
* @param fileName
* @return
*/
private Appender getFileAppender(String filterDestination, String fileName) {
if (!logAppenderRepository.hasAppender(filterDestination)) {

/*
* <appender name="requestLogFileAppender" class="org.apache.log4j.DailyRollingFileAppender">
* <param name="File" value="${org.swisspush.logging.dir}/gateleen-requests.log" />
* <param name="Encoding" value="UTF-8" />
* <param name="Append" value="true" />
* <layout class="org.apache.log4j.EnhancedPatternLayout">
* <param name="ConversionPattern" value="%m%n" />
* </layout>
* </appender>
*/

log.debug("file path: {}", System.getProperty(LOGGING_DIR_PROPERTY) + fileName);

RollingFileAppender.Builder builder = RollingFileAppender.newBuilder().withPolicy(new TimeBasedTriggeringPolicy.Builder().withInterval(1).build());

builder.setName(filterDestination);
if (destinationOptions.containsKey("file")) {
String fileName = destinationOptions.get("file");
RollingFileAppender.Builder builder = RollingFileAppender.newBuilder();
builder.withFileName(System.getProperty(LOGGING_DIR_PROPERTY) + fileName);
builder.withAppend(true);
PatternLayout layout = PatternLayout.createDefaultLayout();
builder.setLayout(layout);
logAppenderRepository.addAppender(filterDestination, builder.build());
builder.withLayout(PatternLayout.createDefaultLayout());
builder.withName(filterDestination);
Appender appender = builder.build();
logAppenderRepository.addAppender(filterDestination, appender);
return appender;
}

return logAppenderRepository.getAppender(filterDestination);
return null;
}

public void setResponse(HttpClientResponse response) {
Expand Down
38 changes: 18 additions & 20 deletions gateleen-logging/src/main/java/org/swisspush/gateleen/logging/RequestPropertyFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.swisspush.gateleen.logging;


import io.vertx.core.MultiMap;
import io.vertx.core.http.HttpServerRequest;
import org.slf4j.Logger;
Expand All @@ -12,6 +11,8 @@
/**
* Class RequestPropertyFilter provides methods to filterProperty requests.
*
* Updated to accept precompiled regex patterns.
*
* @author https://github.com/mcweba [Marc-Andre Weber]
*/
public class RequestPropertyFilter {
Expand All @@ -20,66 +21,63 @@ public class RequestPropertyFilter {
public static final String METHOD = "method";

/**
* Check the provided request against the filterProperty values (key, value) and return a {@link FilterResult} defining
* Check the provided request against the filterProperty values (key, pattern) and return a {@link FilterResult} defining
* whether to filterProperty the request or not.
*
* @param request the request to be checked to filterProperty or not
* @param filterPropertyKey the key of the filterProperty e.g. url, method
* @param filterPropertyValue the value of the filterProperty
* @param filterPropertyPattern the precompiled regex pattern of the filterProperty
* @param reject boolean value from the filterProperty entry called "reject"
* @return the {@link FilterResult} for the provided request
*/
public static FilterResult filterProperty(HttpServerRequest request, String filterPropertyKey, String filterPropertyValue, boolean reject) {

public static FilterResult filterProperty(HttpServerRequest request, String filterPropertyKey, Pattern filterPropertyPattern, boolean reject) {
MultiMap headers = MultiMap.caseInsensitiveMultiMap();
headers.setAll(request.headers());

if (URL.equals(filterPropertyKey)) {
boolean matches = filterRequestURL(request, filterPropertyValue);
boolean matches = filterRequestURL(request, filterPropertyPattern);
FilterResult result = rejectIfNeeded(reject, matches);
logFilterResult(request, filterPropertyKey, filterPropertyValue, result);
logFilterResult(request, filterPropertyKey, filterPropertyPattern.pattern(), result);
return result;
}
if (METHOD.equals(filterPropertyKey)) {
boolean matches = filterRequestMethod(request, filterPropertyValue);
boolean matches = filterRequestMethod(request, filterPropertyPattern);
FilterResult result = rejectIfNeeded(reject, matches);
logFilterResult(request, filterPropertyKey, filterPropertyValue, result);
logFilterResult(request, filterPropertyKey, filterPropertyPattern.pattern(), result);
return result;
}
if (headers.names().contains(filterPropertyKey) && headers.get(filterPropertyKey).equalsIgnoreCase(filterPropertyValue)) {
if (headers.names().contains(filterPropertyKey) && filterPropertyPattern.matcher(headers.get(filterPropertyKey)).matches()) {
FilterResult result = reject ? FilterResult.REJECT : FilterResult.FILTER;
logFilterResult(request, filterPropertyKey, filterPropertyValue, result);
logFilterResult(request, filterPropertyKey, filterPropertyPattern.pattern(), result);
return result;
}
logFilterResult(request, filterPropertyKey, filterPropertyValue, FilterResult.REJECT, true);
logFilterResult(request, filterPropertyKey, filterPropertyPattern.pattern(), FilterResult.REJECT, true);
return FilterResult.REJECT;
}

private static FilterResult rejectIfNeeded(boolean reject, boolean matches) {
if(!matches){
if (!matches) {
return FilterResult.NO_MATCH;
}
return reject ? FilterResult.REJECT : FilterResult.FILTER;
}

private static boolean filterRequestURL(HttpServerRequest request, String url) {
Pattern urlPattern = Pattern.compile(url);
private static boolean filterRequestURL(HttpServerRequest request, Pattern urlPattern) {
Matcher urlMatcher = urlPattern.matcher(request.uri());
return urlMatcher.matches();
}

private static boolean filterRequestMethod(HttpServerRequest request, String method) {
Pattern methodPattern = Pattern.compile(method);
private static boolean filterRequestMethod(HttpServerRequest request, Pattern methodPattern) {
Matcher methodMatcher = methodPattern.matcher(request.method().toString());
return methodMatcher.matches();
}

private static void logFilterResult(HttpServerRequest request, String filterPropertyKey, String filterPropertyValue, FilterResult filterResult){
private static void logFilterResult(HttpServerRequest request, String filterPropertyKey, String filterPropertyValue, FilterResult filterResult) {
logFilterResult(request, filterPropertyKey, filterPropertyValue, filterResult, false);
}

private static void logFilterResult(HttpServerRequest request, String filterPropertyKey, String filterPropertyValue, FilterResult filterResult, boolean noMatchingProperty){
if(FilterResult.NO_MATCH != filterResult) {
private static void logFilterResult(HttpServerRequest request, String filterPropertyKey, String filterPropertyValue, FilterResult filterResult, boolean noMatchingProperty) {
if (FilterResult.NO_MATCH != filterResult) {
Logger log = RequestLoggerFactory.getLogger(RequestPropertyFilter.class, request);
if (!log.isInfoEnabled()) return;
StringBuilder sb = new StringBuilder("Request to ").append(request.uri());
Expand Down
Loading
Loading