From b8f99d31964ae19a09666476901258b2f3a18d08 Mon Sep 17 00:00:00 2001 From: Didi Kohen Date: Sun, 22 Dec 2019 17:54:18 +0200 Subject: [PATCH] Exclude fields support fixes (#7) --- Gemfile.lock | 10 +++---- fluent-plugin-masking.gemspec | 2 +- lib/fluent/plugin/filter_masking.rb | 16 +++++++---- lib/fluent/plugin/helpers.rb | 17 ++++++++++++ lib/fluent/plugin/version.rb | 2 +- test/test_filter_masking.rb | 11 ++++++++ test/test_helpers.rb | 42 +++++++++++++++++++++++++++++ 7 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 lib/fluent/plugin/helpers.rb create mode 100644 test/test_helpers.rb diff --git a/Gemfile.lock b/Gemfile.lock index 675ae28..ee56853 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-masking (1.1.0) + fluent-plugin-masking (1.1.1) fluentd (>= 0.14.0) GEM @@ -10,7 +10,7 @@ GEM concurrent-ruby (1.1.5) cool.io (1.5.4) dig_rb (1.0.1) - fluentd (1.7.4) + fluentd (1.8.0) cool.io (>= 1.4.5, < 2.0.0) dig_rb (~> 1.0.0) http_parser.rb (>= 0.5.1, < 0.7.0) @@ -18,7 +18,7 @@ GEM serverengine (>= 2.0.4, < 3.0.0) sigdump (~> 0.2.2) strptime (>= 0.2.2, < 1.0.0) - tzinfo (~> 2.0) + tzinfo (>= 1.0, < 3.0) tzinfo-data (~> 1.0) yajl-ruby (~> 1.0) http_parser.rb (0.6.0) @@ -45,11 +45,11 @@ PLATFORMS ruby DEPENDENCIES - bundler + bundler (= 1.17.3) fluent-plugin-masking! rake (~> 12.0) test-unit (>= 3.1.0) test-unit-rr BUNDLED WITH - 2.0.2 + 1.17.3 diff --git a/fluent-plugin-masking.gemspec b/fluent-plugin-masking.gemspec index cb6c97b..419c4fe 100644 --- a/fluent-plugin-masking.gemspec +++ b/fluent-plugin-masking.gemspec @@ -21,7 +21,7 @@ Gem::Specification.new do |spec| spec.required_ruby_version = '>= 2.1' spec.add_runtime_dependency "fluentd", ">= 0.14.0" - spec.add_development_dependency "bundler" + spec.add_development_dependency "bundler", "1.17.3" spec.add_development_dependency "rake", "~> 12.0" spec.add_development_dependency "test-unit", ">= 3.1.0" spec.add_development_dependency "test-unit-rr" diff --git a/lib/fluent/plugin/filter_masking.rb b/lib/fluent/plugin/filter_masking.rb index 37bf36e..45d60a8 100644 --- a/lib/fluent/plugin/filter_masking.rb +++ b/lib/fluent/plugin/filter_masking.rb @@ -1,8 +1,10 @@ require 'fluent/filter' +require './lib/fluent/plugin/helpers.rb' module Fluent module Plugin class MaskingFilter < Filter + include Helpers Fluent::Plugin.register_filter("masking", self) # for "@type masking" in configuration MASK_STRING = "*******" @@ -16,13 +18,17 @@ def strToHash(str) def maskRecord(record) maskedRecord = record excludedFields = [] - @fieldsToExcludeJSONPathsArray.each do | field | - field_value = record.dig(*field) - if field_value != nil - excludedFields = excludedFields + field_value.split(',') + begin + @fieldsToExcludeJSONPathsArray.each do | field | + field_value = myDig(record, field) + if field_value != nil + excludedFields = excludedFields + field_value.split(',') + end end + rescue Exception => e + $log.error "Failed to find mask exclude record: #{e}" end - begin + begin recordStr = record.to_s @fields_to_mask_regex.each do | fieldToMaskRegex, fieldToMaskRegexStringReplacement | if !(excludedFields.include? @fields_to_mask_keys[fieldToMaskRegex]) diff --git a/lib/fluent/plugin/helpers.rb b/lib/fluent/plugin/helpers.rb new file mode 100644 index 0000000..6eb6588 --- /dev/null +++ b/lib/fluent/plugin/helpers.rb @@ -0,0 +1,17 @@ +module Helpers + def myDig(input, path) + curr = input + for segment in path do + if curr != nil && curr.is_a?(Hash) + if curr[segment] == nil # segment is not a symbol + curr = curr[segment.to_s] # segment as string + else + curr = curr[segment] # segment as symbol + end + else + return nil + end + end + curr + end +end \ No newline at end of file diff --git a/lib/fluent/plugin/version.rb b/lib/fluent/plugin/version.rb index 6be912c..3518fdc 100644 --- a/lib/fluent/plugin/version.rb +++ b/lib/fluent/plugin/version.rb @@ -1,3 +1,3 @@ module FilterMasking - VERSION = "1.1.0" + VERSION = "1.1.1" end \ No newline at end of file diff --git a/test/test_filter_masking.rb b/test/test_filter_masking.rb index 58df795..3fe3d51 100644 --- a/test/test_filter_masking.rb +++ b/test/test_filter_masking.rb @@ -132,5 +132,16 @@ def filter(config, messages) filtered_records = filter(conf, messages) assert_equal(expected, filtered_records) end + test 'mask field in json string with exclude' do + conf = CONFIG + messages = [ + { :body => "{\"first_name\":\"mickey\",\"last_name\":\"the-dog\", \"type\":\"puggle\"}", :excludedField=>"first_name" } + ] + expected = [ + { :body => "{\"first_name\":\"mickey\",\"last_name\":\"*******\", \"type\":\"puggle\"}", :excludedField=>"first_name" } + ] + filtered_records = filter(conf, messages) + assert_equal(expected, filtered_records) + end end end \ No newline at end of file diff --git a/test/test_helpers.rb b/test/test_helpers.rb new file mode 100644 index 0000000..5dc8358 --- /dev/null +++ b/test/test_helpers.rb @@ -0,0 +1,42 @@ +require "test/unit" +require "./lib/fluent/plugin/helpers.rb" + +class HelpersTest < Test::Unit::TestCase + m = Class.new do + include Helpers + end.new + sub_test_case "myDig function" do + test "Call function with nil" do + t = m.myDig(nil ,[:a]) + assert_equal(t, nil) + end + test "Not found" do + t = m.myDig({:b => "t"},[:a]) + assert_equal(t, nil) + end + test "Found symbol" do + t = m.myDig({:a => "t"},[:a]) + assert_equal(t, "t") + end + test "Found string when given symbol" do + t = m.myDig({"a" => "t"},[:a]) + assert_equal(t, "t") + end + test "Found symbol nested" do + t = m.myDig({:a => {:b => "t"}},[:a, :b]) + assert_equal(t, "t") + end + test "Found string when given symbol nested" do + t = m.myDig({"a" => {"b" => "t"}},[:a, :b]) + assert_equal(t, "t") + end + test "Found hybrid string/symbol when given symbol nested" do + t = m.myDig({"a" => {:b => "t"}},[:a, :b]) + assert_equal(t, "t") + end + test "Does not dig in string" do + t = m.myDig({"a" => {:b => "t"}},[:a, :b, :c]) + assert_equal(t, nil) + end + end +end \ No newline at end of file