Fix log group field (#13)

* remove Gemfile.lock from gitignore * commit Gemfile.lock * add logic to map log group name to log type * increase version to 1.1.4 * add script for generating offline pack
cloud-gov · Dec 10, 2024 · 8e58ed6 · 8e58ed6
1 parent 0700cd6
commit 8e58ed6
Show file tree

Hide file tree

Showing 5 changed files with 232 additions and 6 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,5 @@
 *.gem
-Gemfile.lock
 .bundle
 vendor
 coverage/
-.vscode
+.vscode
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -0,0 +1,198 @@
+PATH
+  remote: .
+  specs:
+    logstash-input-cloudwatch_logs (1.1.3)
+      jar-dependencies (= 0.4.1)
+      logstash-core-plugin-api (>= 1.60, <= 2.99)
+      logstash-integration-aws (>= 7.1.0)
+      stud (~> 0.0.22)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    aws-eventstream (1.3.0)
+    aws-partitions (1.1018.0)
+    aws-sdk-cloudfront (1.107.0)
+      aws-sdk-core (~> 3, >= 3.210.0)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-cloudwatch (1.108.0)
+      aws-sdk-core (~> 3, >= 3.210.0)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-core (3.214.0)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.992.0)
+      aws-sigv4 (~> 1.9)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-kms (1.96.0)
+      aws-sdk-core (~> 3, >= 3.210.0)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-resourcegroups (1.76.0)
+      aws-sdk-core (~> 3, >= 3.210.0)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-s3 (1.176.0)
+      aws-sdk-core (~> 3, >= 3.210.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-sns (1.92.0)
+      aws-sdk-core (~> 3, >= 3.210.0)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-sqs (1.89.0)
+      aws-sdk-core (~> 3, >= 3.210.0)
+      aws-sigv4 (~> 1.5)
+    aws-sigv4 (1.10.1)
+      aws-eventstream (~> 1, >= 1.0.2)
+    chronic_duration (0.10.6)
+      numerizer (~> 0.1.1)
+    clamp (0.6.5)
+    coderay (1.1.3)
+    concurrent-ruby (1.3.4)
+    elasticsearch (5.0.5)
+      elasticsearch-api (= 5.0.5)
+      elasticsearch-transport (= 5.0.5)
+    elasticsearch-api (5.0.5)
+      multi_json
+    elasticsearch-transport (5.0.5)
+      faraday
+      multi_json
+    et-orbi (1.2.11)
+      tzinfo
+    faraday (2.12.2)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-net_http (3.4.0)
+      net-http (>= 0.5.0)
+    ffi (1.17.0-java)
+    filesize (0.0.4)
+    fugit (1.11.1)
+      et-orbi (~> 1, >= 1.2.11)
+      raabro (~> 1.4)
+    gems (0.8.3)
+    i18n (0.6.9)
+    jar-dependencies (0.4.1)
+    jmespath (1.6.2)
+    jrjackson (0.4.20-java)
+    jruby-openssl (0.9.19-java)
+    json (2.9.0-java)
+    language_server-protocol (3.17.0.3)
+    logger (1.6.2)
+    logstash-codec-json (3.0.5)
+      logstash-core-plugin-api (>= 1.60, <= 2.99)
+    logstash-codec-plain (3.0.6)
+      logstash-core-plugin-api (>= 1.60, <= 2.99)
+    logstash-core (5.6.4-java)
+      chronic_duration (= 0.10.6)
+      clamp (~> 0.6.5)
+      concurrent-ruby (~> 1.0, >= 1.0.5)
+      elasticsearch (~> 5.0, >= 5.0.4)
+      filesize (= 0.0.4)
+      gems (~> 0.8.3)
+      i18n (= 0.6.9)
+      jar-dependencies
+      jrjackson (~> 0.4.3)
+      jruby-openssl (= 0.9.19)
+      manticore (>= 0.5.4, < 1.0.0)
+      minitar (~> 0.5.4)
+      pry (~> 0.10.1)
+      puma (~> 2.16)
+      rack (= 1.6.6)
+      ruby-maven (~> 3.3.9)
+      rubyzip (~> 1.1.7)
+      sinatra (~> 1.4, >= 1.4.6)
+      stud (~> 0.0.19)
+      thread_safe (~> 0.3.5)
+      treetop (< 1.5.0)
+    logstash-core-plugin-api (2.1.28-java)
+      logstash-core (= 5.6.4)
+    logstash-integration-aws (7.1.8-java)
+      aws-sdk-cloudfront
+      aws-sdk-cloudwatch
+      aws-sdk-core (~> 3)
+      aws-sdk-resourcegroups
+      aws-sdk-s3
+      aws-sdk-sns
+      aws-sdk-sqs
+      concurrent-ruby
+      logstash-codec-json
+      logstash-codec-plain
+      logstash-core-plugin-api (>= 2.1.12, <= 2.99)
+      rexml
+      rufus-scheduler (>= 3.0.9)
+      stud (~> 0.0.22)
+    manticore (0.9.1-java)
+      openssl_pkcs8_pure
+    method_source (0.8.2)
+    minitar (0.5.4)
+    multi_json (1.15.0)
+    net-http (0.6.0)
+      uri
+    numerizer (0.1.1)
+    openssl_pkcs8_pure (0.0.0.2)
+    parallel (1.26.3)
+    parser (3.3.6.0)
+      ast (~> 2.4.1)
+      racc
+    polyglot (0.3.5)
+    pry (0.10.4-java)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+      spoon (~> 0.0)
+    puma (2.16.0-java)
+    raabro (1.4.0)
+    racc (1.8.1-java)
+    rack (1.6.6)
+    rack-protection (1.5.5)
+      rack
+    rainbow (3.1.1)
+    regexp_parser (2.9.3)
+    rexml (3.3.9)
+    rubocop (1.69.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.36.2, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.36.2)
+      parser (>= 3.3.1.0)
+    ruby-maven (3.3.13)
+      ruby-maven-libs (~> 3.3.9)
+    ruby-maven-libs (3.3.9)
+    ruby-progressbar (1.13.0)
+    rubyzip (1.1.7)
+    rufus-scheduler (3.9.2)
+      fugit (~> 1.1, >= 1.11.1)
+    sinatra (1.4.8)
+      rack (~> 1.5)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    slop (3.6.0)
+    spoon (0.0.6)
+      ffi
+    stud (0.0.23)
+    thread_safe (0.3.6-java)
+    tilt (2.4.0)
+    treetop (1.4.15)
+      polyglot
+      polyglot (>= 0.3.1)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (3.1.2)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
+    uri (1.0.2)
+
+PLATFORMS
+  universal-java-17
+
+DEPENDENCIES
+  logstash-input-cloudwatch_logs!
+  rubocop
+
+BUNDLED WITH
+   2.3.26
diff --git a/bin/create-offline-pack.sh b/bin/create-offline-pack.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+LOGSTASH_PLUGIN=$1
+if [[ -z "$LOGSTASH_PLUGIN" ]]; then
+  echo "Path to logstash plugin required as first argument"
+  exit 1
+fi
+
+GEM_NAME="logstash-input-cloudwatch_logs"
+VERSION=$(grep 's.version' logstash-input-cloudwatch_logs.gemspec | awk '{print $3}' | tr -d "'")
+
+jruby -S gem build "$GEM_NAME.gemspec"
+$LOGSTASH_PLUGIN install "$GEM_NAME-$VERSION.gem"
+$LOGSTASH_PLUGIN prepare-offline-pack --output "$GEM_NAME-$VERSION.zip" "$GEM_NAME"
diff --git a/lib/logstash/inputs/cloudwatch_logs.rb b/lib/logstash/inputs/cloudwatch_logs.rb
@@ -165,8 +165,23 @@ def priority_of(group)
     @priority.index(group) || -1
   end
 
+  def map_group_to_log_type(group)
+    case group
+    when %r{/aws/rds/.*}
+      'rds'
+    when %r{/aws/OpenSearchService/.*}i
+      'opensearch'
+    when %r{/aws/ElasticCache/.*}i
+      'elasticache'
+    else
+      'unknown log type'
+    end
+  end
+
   def process_group(group)
     next_token = nil
+    log_type = map_group_to_log_type(group)
+
     loop do
       @sincedb[group] = 0 unless @sincedb.member?(group)
 
@@ -178,7 +193,7 @@ def process_group(group)
       }
       resp = @cloudwatch.filter_log_events(params)
       resp.events.each do |event|
-        process_log(event, group)
+        process_log(event, group, log_type)
       end
 
       _sincedb_write
@@ -222,7 +237,7 @@ def fetch_tags_from_cloudwatch(log_group_name)
     tags
   end
 
-  def process_log(log, group)
+  def process_log(log, group, log_type)
     tags = fetch_tags(group)
 
     @logger.debug("processing_log #{log}")
@@ -233,7 +248,7 @@ def process_log(log, group)
       event.set('[cloudwatch_logs][log_stream]', log.log_stream_name)
       event.set('[cloudwatch_logs][event_id]', log.event_id)
       event.set('[cloudwatch_logs][tags]', tags)
-      event.set('[cloudwatch_logs][log_group_prefix]', @log_group_prefix)
+      event.set('[cloudwatch_logs][log_type]', log_type)
       decorate(event)
 
       @queue << event

diff --git a/logstash-input-cloudwatch_logs.gemspec b/logstash-input-cloudwatch_logs.gemspec
@@ -4,7 +4,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '= 3.1.4'
 
   s.name            = 'logstash-input-cloudwatch_logs'
-  s.version         = '1.1.3'
+  s.version         = '1.1.4'
   s.licenses        = ['Apache-2.0']
   s.summary         = 'Stream events from CloudWatch Logs.'
   s.description     = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline'\