Merge pull request #961 from smileusd/upstream_add_black_list_in_log_…

…watcher

add black list to aviod take too much efforts to translate in file log watcher

config/disk-log-message-filelog.json

@@ -0,0 +1,28 @@
+{
+	"plugin": "filelog",
+	"pluginConfig": {
+		"timestamp": "^.{15}",
+		"message": "(?i)Currently unreadable.*sectors|(?i)Offline uncorrectable sectors",
+		"timestampFormat": "Jan _2 15:04:05"
+	},
+	"logPath": "/var/log/messages",
+	"lookback": "10h",
+	"bufferSize": 1,
+	"source": "disk-monitor",
+	"skipList": [ " audit:", " audit[" ],
+	"conditions": [
+		{
+			"type": "DiskBadBlock",
+			"reason": "DiskBadBlock",
+			"message": "Disk no bad block"
+		},
+	],
+	"rules": [
+		{
+			"type": "permanent",
+			"condition": "DiskBadBlock",
+			"reason": "DiskBadBlock",
+			"pattern": ".*([1-9]\\d{2,}) (Currently unreadable.*sectors|Offline uncorrectable sectors).*"
+		},
+	]
+}

pkg/systemlogmonitor/logwatchers/filelog/log_watcher.go

@@ -116,6 +116,9 @@ func (s *filelogWatcher) watchLoop() {
 		}
 		line = buffer.String()
 		buffer.Reset()
+		if s.filterSkipList(line) {
+			continue
+		}
 		log, err := s.translator.translate(strings.TrimSuffix(line, "\n"))
 		if err != nil {
 			klog.Warningf("Unable to parse line: %q, %v", line, err)
@@ -129,3 +132,12 @@ func (s *filelogWatcher) watchLoop() {
 		s.logCh <- log
 	}
 }
+
+func (s *filelogWatcher) filterSkipList(line string) bool {
+	for _ , skipItem := range s.cfg.SkipList {
+		if strings.Contains(line, skipItem) {
+			return true
+		}
+	}
+	return false
+}

pkg/systemlogmonitor/logwatchers/filelog/log_watcher_test.go

@@ -176,3 +176,36 @@ Jan  2 03:04:05 kernel: [2.000000] 3
 		}
 	}
 }
+
+func TestFilterSkipList(t *testing.T) {
+	s := &filelogWatcher{
+		cfg: types.WatcherConfig{
+			SkipList: []string{
+				" audit:", " kubelet:",
+			},
+		},
+	}
+	testcase := []struct{
+		log string
+		expect bool
+	}{
+		{
+			log: `Jan  2 03:04:03 kernel: [0.000000] 1`,
+			expect: false,
+		},
+		{
+			log: `Jan  2 03:04:04 audit: [1.000000] 2`,
+			expect: true,
+		},
+		{
+			log: `Jan  2 03:04:05 kubelet: [2.000000] 3`,
+			expect: true,
+
+		},
+	}
+	for i, test := range testcase {
+		if s.filterSkipList(test.log) != test.expect {
+			t.Errorf("test case %d: expect %v but got %v", i, test.expect, s.filterSkipList(test.log))
+		}
+	}
+}

pkg/systemlogmonitor/logwatchers/types/log_watcher.go

@@ -36,6 +36,8 @@ type WatcherConfig struct {
 	// PluginConfig is a key/value configuration of a plugin. Valid configurations
 	// are defined in different log watcher plugin.
 	PluginConfig map[string]string `json:"pluginConfig,omitempty"`
+	// Skip the log lines containing any of the strings in the list to avoid running unnecessary regex.
+	SkipList []string `json:"skipList,omitempty"`
 	// LogPath is the path to the log
 	LogPath string `json:"logPath,omitempty"`
 	// Lookback is the time log watcher looks up