Added support for processing Apache logs

This will be abstracted into a filter at some point but the need for processing these logs on Linux was required for me at the moment
Struck713 · Jan 12, 2023 · 48a17ae · 48a17ae
1 parent ea1d917
commit 48a17ae
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 9 deletions.
diff --git a/src/main/java/com/noah/syslog/SyslogAgent.java b/src/main/java/com/noah/syslog/SyslogAgent.java
@@ -35,10 +35,10 @@ public static void main(String[] args) throws IOException, InterruptedException
 
         SyslogAgent.LOGGER.info("Detecting operating system..");
         OSUtil.Types osType = OSUtil.getType();
+        SyslogAgent.LOGGER.info("Detected operating system: " + osType.name());
         if (osType == OSUtil.Types.WINDOWS) logAdapter = new WindowsAdapter(config.getSources(), config.getFilters());
         else if (osType == OSUtil.Types.UNIX) logAdapter = new LinuxAdapter(timeBetweenReads, config.getSources(), config.getFilters());
-        else { SyslogAgent.LOGGER.info("Operating system, " + osType.name() + " is unsupported!"); return; }
-        SyslogAgent.LOGGER.info("Detected operating system: " + osType.name());
+        else { SyslogAgent.LOGGER.error("Operating system, " + osType.name() + " is unsupported!"); return; }
 
         SyslogAgent.LOGGER.info("Initializing SyslogAgent..");
         ConfigHost host = config.getHost();

diff --git a/src/main/java/com/noah/syslog/log/adapter/LinuxAdapter.java b/src/main/java/com/noah/syslog/log/adapter/LinuxAdapter.java
@@ -1,5 +1,6 @@
 package com.noah.syslog.log.adapter;
 
+import com.google.gson.reflect.TypeToken;
 import com.noah.syslog.SyslogAgent;
 import com.noah.syslog.config.ConfigFilter;
 import com.noah.syslog.log.LogAdapter;
@@ -9,8 +10,11 @@
 import com.noah.syslog.log.watcher.FileWatcherListener;
 import com.noah.syslog.message.enums.Priority;
 import com.noah.syslog.message.enums.Severity;
+import com.noah.syslog.util.FileUtil;
+import com.noah.syslog.util.StringUtil;
 
 import java.io.*;
+import java.lang.reflect.Type;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.*;
@@ -19,7 +23,8 @@
 
 public class LinuxAdapter implements LogAdapter, FileWatcherListener {
 
-    public static final SimpleDateFormat APACHE_DATE_FORMAT = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss Z");
+    public static final SimpleDateFormat APACHE_DATE_FORMAT = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ssZ");
+    public static final File STATES = new File("states.json");
 
     private FileWatcher watcher;
     private Map<File, Long> fileStates;
@@ -29,9 +34,16 @@ public class LinuxAdapter implements LogAdapter, FileWatcherListener {
     public LinuxAdapter(long interval, List<String> sources, List<ConfigFilter> filters) {
         this.watcher = new FileWatcher(interval, this);
         this.filters = filters.stream().map(Filter::of).collect(Collectors.toList());
-        this.fileStates = new HashMap<>();
         this.items = new ArrayList<>();
 
+        this.fileStates = new HashMap<>();
+        if (LinuxAdapter.STATES.exists()) {
+            String rawJson = FileUtil.read(LinuxAdapter.STATES);
+            Type mapType = new TypeToken<Map<String, Long>>() {}.getType();
+            Map<String, Long> stringsMap = SyslogAgent.GSON.fromJson(rawJson, mapType);
+            stringsMap.forEach((k, v) -> this.fileStates.put(new File(k), v));
+        }
+
         sources.stream()
             .map(File::new)
             .filter(file -> {
@@ -40,6 +52,8 @@ public LinuxAdapter(long interval, List<String> sources, List<ConfigFilter> filt
                 return exists;
             })
             .forEach(this.watcher::watch);
+
+        this.watcher.start();
     }
 
     private int messageId = 0;
@@ -50,10 +64,10 @@ public List<LogItem> next() {
         List<LogItem> logItems = new ArrayList<>();
         this.items.stream()
             .filter(item -> item.contains("404"))
-            .map(item -> item.split(" ")).forEach(list -> {
-
+            .map(item -> item.split(" "))
+            .forEach(list -> {
                 String ip = list[0];
-                String request = list[5] + list[6] + list[7];
+                String request = StringUtil.join(" ", list[5], list[6], list[7]);
                 String datePart1 = list[3];
                 String datePart2 = list[4];
                 String dateString = datePart1.substring(1) + datePart2.substring(0, datePart2.length() - 1);
@@ -88,7 +102,12 @@ public void onFileChange(File file) {
                 this.items.add(string);
                 read.getAndIncrement();
             });
-            this.fileStates.put(file, state + read.get());
+            long newState = read.get();
+            this.fileStates.put(file, state + newState);
+
+            SyslogAgent.LOGGER.info("Read " + newState + " new lines from: " + file.getAbsolutePath());
+            String rawStates = SyslogAgent.GSON.toJson(this.fileStates);
+            FileUtil.write(rawStates, LinuxAdapter.STATES);
         } catch (IOException e) {
             SyslogAgent.LOGGER.error("Failed to read from file: " + file.getName());
         }

diff --git a/src/main/java/com/noah/syslog/log/watcher/FileWatcher.java b/src/main/java/com/noah/syslog/log/watcher/FileWatcher.java
@@ -2,15 +2,24 @@
 
 import com.noah.syslog.SyslogAgent;
 import com.noah.syslog.log.adapter.LinuxAdapter;
+import com.noah.syslog.log.filters.Filter;
 import org.apache.commons.io.monitor.FileAlterationListener;
 import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
 import org.apache.commons.io.monitor.FileAlterationMonitor;
 import org.apache.commons.io.monitor.FileAlterationObserver;
 
 import java.io.File;
+import java.io.FileFilter;
 
 public class FileWatcher {
 
+    private static final FileFilter FILTER = new FileFilter() {
+        @Override
+        public boolean accept(File pathname) {
+            return pathname.getName().endsWith(".log");
+        }
+    };
+
     private FileAlterationMonitor monitor;
     private FileWatcherListener listener;
 
@@ -20,14 +29,16 @@ public FileWatcher(long interval, FileWatcherListener listener) {
     }
 
     public void watch(File file) {
-        FileAlterationObserver observer = new FileAlterationObserver(file);
+        SyslogAgent.LOGGER.info("File watcher was started for: " + file.getAbsolutePath());
+        FileAlterationObserver observer = new FileAlterationObserver(file, FileWatcher.FILTER);
         observer.addListener(new FileAlterationListenerAdaptor() {
             @Override
             public void onFileChange(File file) {
                 listener.onFileChange(file);
             }
         });
         this.monitor.addObserver(observer);
+        for (File item : file.listFiles(FileWatcher.FILTER)) this.listener.onFileChange(item);
     }
 
     public void stop() {