Merge pull request #11 from PayU/enhancements

feat: add case insensitive configuration support
PayU · Mar 14, 2021 · 989c44d · 989c44d
2 parents b8346da + b7125a3
commit 989c44d
Show file tree

Hide file tree

Showing 12 changed files with 132 additions and 103 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -3,15 +3,9 @@ orbs:
   ruby: circleci/ruby@0.1.2 
 
 executors:
-  v2-6-3:
-    docker:
-      - image: circleci/ruby:2.6.3-stretch-node
   v2-5-0:
     docker:
-      - image: circleci/ruby:2.5.0-stretch-node
-  v2-3-8:
-    docker:
-      - image: circleci/ruby:2.3.8-stretch-node
+      - image: circleci/ruby:2.5.0
 
 jobs:
   tests:
@@ -21,13 +15,15 @@ jobs:
     executor: << parameters.ruby-version >>
     steps:
       - checkout
-      - run: gem update bundler
+      - run: gem install bundler
+      - run: bundle install
+      - run: ruby -r ./test/*.rb
 
 workflows:
   tests:
     jobs:
       - tests:
           matrix:
             parameters:
-              ruby-version: [v2-6-3, v2-5-0, v2-3-8]
+              ruby-version: [v2-5-0]
 
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,6 @@
 .bundle/
 vendor/
 *.gem
-.idea/
+.idea/
+Gemfile.lock
+Rakefile
diff --git a/.travis.yml b/.travis.yml
diff --git a/Gemfile.lock b/Gemfile.lock
diff --git a/README.md b/README.md
@@ -8,7 +8,7 @@ Fluentd filter plugin to mask sensitive or privacy records with `*******` in pla
 ## Requirements
 | fluent-plugin-masking    | fluentd    | ruby   |
 | ---------------------    | ---------- | ------ |
-| 1.0.x                    | 	>= v0.14.0 | >= 2.1 |
+| 1.2.x                    | 	>= v0.14.0 | >= 2.5 |
 
 
 ## Installation
@@ -35,10 +35,9 @@ Example fields-to-mask-file:
 ```
 name
 email
-phone
+phone/i # the '/i' suffix will make sure phone field will be case insensitive
 ```
 
-
 ## Quick Guide
 
 ### Configuration:
@@ -98,3 +97,11 @@ echo '{ :body => "{\"first_name\":\"mickey\", \"type\":\"puggle\", \"last_name\"
 ```
 2019-12-01 14:25:53.385681000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"mickey\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"the-dog\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
 ```
+
+
+### Run Unit Tests
+```
+gem install bundler
+bundle install
+ruby -r ./test/*.rb
+```
diff --git a/Rakefile b/Rakefile
diff --git a/fluent-plugin-masking.gemspec b/fluent-plugin-masking.gemspec
@@ -18,11 +18,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.license = "Apache-2.0"
 
-  spec.required_ruby_version = '>= 2.1'
+  spec.required_ruby_version = '>= 2.5.0'
 
   spec.add_runtime_dependency "fluentd", ">= 0.14.0"
-  spec.add_development_dependency "bundler", "1.17.3"
-  spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", ">= 3.1.0"
   spec.add_development_dependency "test-unit-rr"
 end
diff --git a/lib/fluent/plugin/filter_masking.rb b/lib/fluent/plugin/filter_masking.rb
@@ -73,19 +73,27 @@ def configure(conf)
 
         File.open(fieldsToMaskFilePath, "r") do |f|
           f.each_line do |line|
-
             value = line.to_s # make sure it's string
             value = value.gsub(/\s+/, "") # remove spaces
             value = value.gsub('\n', '') # remove line breakers
 
+            if value.end_with? "/i"
+              # case insensitive
+              value = value.delete_suffix('/i')
+              hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/mi)
+              innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"\s*:\s*(\\+|\{).+?((?=(})|,( *|)(\s|\\+)\"(}*))|(?=}"$)|("}(?!\"|\\)))/mi)
+            else
+              # case sensitive
+              hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m)
+              innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"\s*:\s*(\\+|\{).+?((?=(})|,( *|)(\s|\\+)\"(}*))|(?=}"$)|("}(?!\"|\\)))/m)
+            end
+
             @fields_to_mask.push(value)
 
-            hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m) # mask element in hash object
             hashObjectRegexStringReplacement = ":#{value}=>\"#{MASK_STRING}\""
             @fields_to_mask_regex[hashObjectRegex] = hashObjectRegexStringReplacement
             @fields_to_mask_keys[hashObjectRegex] = value
 
-            innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/m) # mask element in json string using capture groups that count the level of escaping inside the json string
             innerJSONStringRegexStringReplacement = "\\1\"#{value}\\1\":\\1\"#{MASK_STRING}\\1\""
             @fields_to_mask_regex[innerJSONStringRegex] = innerJSONStringRegexStringReplacement
             @fields_to_mask_keys[innerJSONStringRegex] = value

diff --git a/lib/fluent/plugin/version.rb b/lib/fluent/plugin/version.rb
@@ -1,3 +1,3 @@
 module FilterMasking
-    VERSION = "1.1.2"
-end 
+    VERSION = "1.2.0"
+end
diff --git a/test/fields-to-mask b/test/fields-to-mask
@@ -3,4 +3,5 @@ first_name
 last_name
 street
 number
-password
+password
+json_str_field
diff --git a/test/fields-to-mask-insensitive b/test/fields-to-mask-insensitive
@@ -0,0 +1,5 @@
+email/i
+first_name
+last_name/i
+street/i
+number
diff --git a/test/test_filter_masking.rb b/test/test_filter_masking.rb
@@ -1,5 +1,3 @@
-# test/plugin/test_filter_your_own.rb
-
 require "test-unit"
 require "fluent/test"
 require "fluent/test/driver/filter"
@@ -25,6 +23,11 @@ def setup
     fieldsToMaskFilePath test/fields-to-mask
   ]
 
+  # configuration for tests with case insensitive fields
+  CONFIG_CASE_INSENSITIVE = %[
+    fieldsToMaskFilePath test/fields-to-mask-insensitive
+  ]
+
   def create_driver(conf = CONFIG)
     Fluent::Test::Driver::Filter.new(Fluent::Plugin::MaskingFilter).configure(conf)
   end
@@ -39,7 +42,7 @@ def filter(config, messages)
     d.filtered_records
   end
 
-  sub_test_case 'plugin will mask all fields that need masking' do
+  sub_test_case 'plugin will mask all fields that need masking - case sensitive fields' do
     test 'mask field in hash object' do
       conf = CONFIG_NO_EXCLUDE
       messages = [
@@ -121,6 +124,7 @@ def filter(config, messages)
       filtered_records = filter(conf, messages)
       assert_equal(expected, filtered_records)
     end
+
     test 'mask field in hash object with base and nested exclude' do
       conf = CONFIG
       messages = [
@@ -132,6 +136,7 @@ def filter(config, messages)
       filtered_records = filter(conf, messages)
       assert_equal(expected, filtered_records)
     end
+
     test 'mask field in json string with exclude' do
       conf = CONFIG
       messages = [
@@ -143,5 +148,90 @@ def filter(config, messages)
       filtered_records = filter(conf, messages)
       assert_equal(expected, filtered_records)
     end
+
+    test 'mask field which is inner json string field (should mask the whole object)' do
+      conf = CONFIG
+      messages = [
+        {
+          :body => { 
+            :action_name => "some_action", 
+            :action_type => "some type",
+            :request => {
+              :body_str => "{\"str_field\":\"mickey\",\"json_str_field\": {\"id\":\"ed8a8378-3235-4923-b802-7700167d1870\"},\"not_mask\":\"some_value\"}"
+            }
+          },
+          :timestamp => "2020-06-08T16:00:57.341Z"
+        }
+      ]
+
+      expected = [
+        {
+          :body => { 
+            :action_name => "some_action", 
+            :action_type => "some type",
+            :request => {
+              :body_str => "{\"str_field\":\"mickey\",\"json_str_field\":\"*******\",\"not_mask\":\"some_value\"}"
+            }
+          },
+          :timestamp => "2020-06-08T16:00:57.341Z"
+        }
+      ]
+
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
   end
+
+  sub_test_case 'plugin will mask all fields that need masking - case INSENSITIVE fields' do
+
+    test 'mask field in hash object with camel case' do
+      conf = CONFIG_CASE_INSENSITIVE
+      messages = [
+        {:not_masked_field=>"mickey-the-dog", :Email=>"mickey-the-dog@zooz.com"}
+      ]
+      expected = [
+        {:not_masked_field=>"mickey-the-dog", :email=>MASK_STRING}
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+
+    test 'not mask field in hash object since case not match' do
+      conf = CONFIG_CASE_INSENSITIVE
+      messages = [
+        {:not_masked_field=>"mickey-the-dog", :FIRST_NAME=>"mickey-the-dog@zooz.com"}
+      ]
+      expected = [
+        {:not_masked_field=>"mickey-the-dog", :FIRST_NAME=>"mickey-the-dog@zooz.com"}
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+
+    test 'mask field in hash object with snakecase' do
+      conf = CONFIG_CASE_INSENSITIVE
+      messages = [
+        {:not_masked_field=>"mickey-the-dog", :LaSt_NaMe=>"mickey-the-dog@zooz.com"}
+      ]
+      expected = [
+        {:not_masked_field=>"mickey-the-dog", :last_name=>MASK_STRING}
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+
+    test 'mask case insensitive and case sensitive field in nested json escaped string' do
+      conf = CONFIG_CASE_INSENSITIVE
+      messages = [
+        { :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"the-dog\",\"address\":\"{\\\"Street\":\\\"Austin\\\",\\\"number\":\\\"89\\\"}\", \"type\":\"puggle\"}" } 
+      ]
+      expected = [
+        { :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"*******\",\"address\":\"{\\\"street\\\":\\\"*******\\\",\\\"number\\\":\\\"*******\\\"}\", \"type\":\"puggle\"}" }
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+
+  end
+
 end