[SOAR-17918] Update RuntimeValidator to ignore specific file extensio…

…ns (#190)

* Skipping cache files, unit test, fixing failing tests

* Skipping cache files, unit test, fixing failing tests

* Bumping the version and changelog

* Removing unnecessary object

README.md

@@ -62,6 +62,7 @@ be on your way to contributing!
 
 ## Changelog
 
+* 2.47.21 - `RuntimeValidator` - Fixed issue where validator failed if plugin contained a cache files
 * 2.47.20 - Bumping `requests` in requirements.txt and bumping `ldap` in the `version_pin_validator` requirements.txt
 * 2.47.19 - `VersionBumpValidator` - Fixed issue where validator failed if plugin contained no actions or triggers
 * 2.47.18 - `HelpInputOutputValidator` | `SpecPropertiesValidator` - Update to enable `placeholder` and `tooltip` validation | `RuntimeValidator` - Added SDK version validation

icon_validator/__init__.py

@@ -1 +1 @@
-VERSION = "2.47.20"
+VERSION = "2.47.21"

icon_validator/rules/plugin_validators/runtime_validator.py

@@ -49,6 +49,8 @@ def validate_caching(spec):
             for path in paths:
                 for root, dirs, files in os.walk(path):
                     for file in files:
+                        if file == ".DS_Store" or file.endswith(".pyc"):
+                            continue
                         with open(os.path.join(root, file), "r") as open_file:
                             file_str = open_file.read().replace("\n", "")
 

unit_test/plugin_examples/good_plugin_with_cache/.CHECKSUM

@@ -0,0 +1,23 @@
+{
+	"spec": "7b971f1ccf8214be1b140fe212ade68d",
+	"manifest": "c4fe1b5757c0fbf72a28679cc6303e30",
+	"setup": "9d01465bd990dde0f93e26a9fbda8a33",
+	"schemas": [
+		{
+			"identifier": "decode/schema.py",
+			"hash": "e8816b23112cbb301c9255c29323c4a8"
+		},
+		{
+			"identifier": "encode/schema.py",
+			"hash": "70afbd79bd72035de62b32983ee57ba3"
+		},
+		{
+			"identifier": "connection/schema.py",
+			"hash": "da5382221ca2a33a2f854e17b068d502"
+		},
+		{
+			"identifier": "test_task/schema.py",
+			"hash": "930a8c07873126927dbcaf6d8e5226fa"
+		}
+	]
+}

unit_test/plugin_examples/good_plugin_with_cache/.dockerignore

@@ -0,0 +1,9 @@
+unit_test/**/*
+unit_test
+examples/**/*
+examples
+tests
+tests/**/*
+**/*.json
+**/*.tar
+**/*.gz

unit_test/plugin_examples/good_plugin_with_cache/Dockerfile

@@ -0,0 +1,21 @@
+FROM rapid7/insightconnect-python-3-slim-plugin:6.2.0
+LABEL organization=komand
+LABEL sdk=python
+LABEL type=plugin
+
+ENV SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt
+ENV SSL_CERT_DIR /etc/ssl/certs
+ENV REQUESTS_CA_BUNDLE  /etc/ssl/certs/ca-certificates.crt
+
+ADD ./plugin.spec.yaml /plugin.spec.yaml
+ADD . /python/src
+
+WORKDIR /python/src
+# Add any package dependencies here
+
+# End package dependencies
+RUN if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+RUN python setup.py build && python setup.py install
+
+
+ENTRYPOINT ["/usr/local/bin/komand_base64"]

unit_test/plugin_examples/good_plugin_with_cache/Makefile

@@ -0,0 +1,53 @@
+# Include other Makefiles for improved functionality
+INCLUDE_DIR = ../../tools/Makefiles
+MAKEFILES := $(wildcard $(INCLUDE_DIR)/*.mk)
+# We can't guarantee customers will have the include files
+# - prefix to ignore Makefiles when not present
+# https://www.gnu.org/software/make/manual/html_node/Include.html
+-include $(MAKEFILES)
+
+ifneq ($(MAKEFILES),)
+  $(info [$(YELLOW)*$(NORMAL)] Use ``make menu`` for available targets)
+  $(info [$(YELLOW)*$(NORMAL)] Including available Makefiles: $(MAKEFILES))
+  $(info --)
+else
+  $(warning Makefile includes directory not present: $(INCLUDE_DIR))
+endif
+
+VERSION?=$(shell grep '^version: ' plugin.spec.yaml | sed 's/version: //')
+NAME?=$(shell grep '^name: ' plugin.spec.yaml | sed 's/name: //')
+VENDOR?=$(shell grep '^vendor: ' plugin.spec.yaml | sed 's/vendor: //')
+CWD?=$(shell basename $(PWD))
+_NAME?=$(shell echo $(NAME) | awk '{ print toupper(substr($$0,1,1)) tolower(substr($$0,2)) }')
+PKG=$(VENDOR)-$(NAME)-$(VERSION).tar.gz
+
+# Set default target explicitly. Make's default behavior is the first target in the Makefile.
+# We don't want that behavior due to includes which are read first
+.DEFAULT_GOAL := default # Make >= v3.80 (make -version)
+
+
+default: image tarball
+
+tarball:
+	$(info [$(YELLOW)*$(NORMAL)] Creating plugin tarball)
+	rm -rf build
+	rm -rf $(PKG)
+	tar -cvzf $(PKG) --exclude=$(PKG) --exclude=tests --exclude=run.sh *
+
+image:
+	$(info [$(YELLOW)*$(NORMAL)] Building plugin image)
+	docker build --pull -t $(VENDOR)/$(NAME):$(VERSION) .
+	docker tag $(VENDOR)/$(NAME):$(VERSION) $(VENDOR)/$(NAME):latest
+
+regenerate:
+	$(info [$(YELLOW)*$(NORMAL)] Regenerating schema from plugin.spec.yaml)
+	icon-plugin generate python --regenerate
+
+export: image
+	$(info [$(YELLOW)*$(NORMAL)] Exporting docker image)
+	@printf "\n ---> Exporting Docker image to ./$(VENDOR)_$(NAME)_$(VERSION).tar\n"
+	@docker save $(VENDOR)/$(NAME):$(VERSION) | gzip > $(VENDOR)_$(NAME)_$(VERSION).tar
+
+# Make will not run a target if a file of the same name exists unless setting phony targets
+# https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
+.PHONY: default tarball image regenerate

unit_test/plugin_examples/good_plugin_with_cache/bin/komand_base64

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# GENERATED BY KOMAND SDK - DO NOT EDIT
+import os
+import json
+from sys import argv
+
+Name = "Base64"
+Vendor = "rapid7"
+Version = "1.1.2"
+Description = "Encode and decode data using the base64 alphabet"
+
+
+def main():
+    if 'http' in argv:
+        if os.environ.get("GUNICORN_CONFIG_FILE"):
+            with open(os.environ.get("GUNICORN_CONFIG_FILE")) as gf:
+                gunicorn_cfg = json.load(gf)
+                if gunicorn_cfg.get("worker_class", "sync") == "gevent":
+                    from gevent import monkey
+                    monkey.patch_all()
+        elif 'gevent' in argv:
+            from gevent import monkey
+            monkey.patch_all()
+
+    import insightconnect_plugin_runtime
+    from komand_base64 import connection, actions, triggers, tasks
+
+    class ICONBase64(insightconnect_plugin_runtime.Plugin):
+        def __init__(self):
+            super(self.__class__, self).__init__(
+                name=Name,
+                vendor=Vendor,
+                version=Version,
+                description=Description,
+                connection=connection.Connection()
+            )
+            self.add_action(actions.Decode())
+
+            self.add_action(actions.Encode())
+
+            self.add_task(tasks.TestTask())
+
+
+    """Run plugin"""
+    cli = insightconnect_plugin_runtime.CLI(ICONBase64())
+    cli.run()
+
+
+if __name__ == "__main__":
+    main()

unit_test/plugin_examples/good_plugin_with_cache/help.md

@@ -0,0 +1,173 @@
+# Description
+
+[Base64](https://en.wikipedia.org/wiki/Base64) is a common binary-to-text encoding scheme used in various protocols and software such as MIME to carry data stored in binary formats across channels that only reliably support text content. This plugin allows data to be Base64-encoded or decoded using the standard Base64 alphabet.
+
+# Key Features
+
+* Encode data in Base64 to transfer binary data, image files, etc. in a text format
+* Decode Base64 encoded text to reveal the plaintext
+
+# Requirements
+
+_This plugin does not contain any requirements._
+
+# Supported Product Versions
+
+* 1.0.0
+* 1.0.1
+
+# Documentation
+
+## Setup
+
+_This plugin does not contain a connection._
+## Technical Details
+
+### Actions
+
+#### Encoder
+
+This action is used to encode data to Base64.
+
+##### Input
+
+|Name|Type|Default|Required|Description|Enum|Example|Placeholder|Tooltip|
+|----|----|-------|--------|-----------|----|-------|-----------|-------|
+|content|string|None|True|Data to encode|None|This is a string|None|None|
+
+Example input:
+
+```
+{
+  "content": "This is a string"
+}
+```
+
+##### Output
+
+|Name|Type|Required|Description|Example|
+|----|----|--------|-----------|
+|data|bytes|True|Encoded data result|This is a string|
+
+Example output:
+
+```
+{
+  "errors": "replace"
+}
+```
+
+#### Decoder
+
+This action is used to decode Base64 to data.
+
+##### Input
+
+|Name|Type|Default|Required|Description|Enum|Example|Placeholder|Tooltip|
+|----|----|-------|--------|-----------|----|-------|-----------|-------|
+|base64|bytes|None|True|Data to decode|None|==Md|None|None|
+|errors|string|nothing|False|How errors should be handled when decoding Base64|["replace", "ignore", "nothing"]|replace|None|None|
+
+Example input:
+
+```
+{
+  "base64": "==Md",
+  "errors": "replace"
+}
+```
+
+##### Output
+
+|Name|Type|Required|Description|Example|
+|----|----|--------|-----------|
+|data|string|True|Decoded data result|==Md|
+
+Example output:
+
+```
+{
+  "errors": "replace"
+}
+```
+
+### Triggers
+
+_This plugin does not contain any triggers._
+
+### Tasks
+
+#### Test Task
+
+This task is used to decode Base64 to data.
+
+Supported schedule types for this task include:
+  - cron
+  - minutes
+  - hours
+  - datetime
+
+##### Input
+
+|Name|Type|Default|Required|Description|Enum|Example|Placeholder|Tooltip|
+|----|----|-------|--------|-----------|----|-------|-----------|-------|
+|base64|bytes|None|True|Data to decode|None|1234|None|None|
+|errors|string|nothing|False|How errors should be handled when decoding Base64|["replace", "ignore", "nothing"]|replace|None|None|
+
+Example input:
+
+```
+{
+  "base64": 12345,
+  "errors": "replace"
+}
+```
+
+##### Output
+
+|Name|Type|Required|Description|Example|
+|----|----|--------|-----------|
+|data|string|True|Decoded data result|12345|
+
+Example output:
+
+```
+{
+  "errors": "replace"
+}
+```
+
+### Custom Output Types
+
+_This plugin does not contain any custom output types._
+
+## Troubleshooting
+
+For the Base64 decode action, be sure that the input contains valid Base64 data.
+
+If the Base64 you're decoding contains any non UTF-8 characters the plugin will fail. To remedy this issue, there's a
+option to set how errors are to be handled. These options are "replace" and "ignore". Replace will change all non UTF-8
+characters to `\uffd` or `?`. While ignore will drop the character from the output.
+
+# Version History
+
+* 1.1.2 - New spec and help.md format for the Hub
+* 1.1.1 - Fixed issue where action Decode required error parameter
+* 1.1.0 - Bug fix in decode action, added an option for error handling
+* 1.0.0 - Support web server mode
+* 0.2.2 - Generate plugin with new schema
+* 0.2.1 - SSL bug fix in SDK
+* 0.2.0 - Plugin variable naming and description improvements, add required outputs
+* 0.1.1 - Bug fix in output variables
+* 0.1.0 - Initial plugin
+
+# Links
+
+* [Base64](https://en.wikipedia.org/wiki/Base64)
+
+## References
+
+* [Base64](https://en.wikipedia.org/wiki/Base64)
+* [Python Base64 Encode](https://docs.python.org/2/library/base64.html#base64.standard_b64encode)
+* [Python Base64 Decode](https://docs.python.org/2/library/base64.html#base64.standard_b64decode)
+

unit_test/plugin_examples/good_plugin_with_cache/komand_base64/__init__.py

@@ -0,0 +1 @@
+# GENERATED BY KOMAND SDK - DO NOT EDIT

unit_test/plugin_examples/good_plugin_with_cache/komand_base64/actions/__init__.py

@@ -0,0 +1,3 @@
+# GENERATED BY KOMAND SDK - DO NOT EDIT
+from .decode.action import Decode
+from .encode.action import Encode

unit_test/plugin_examples/good_plugin_with_cache/komand_base64/actions/decode/__init__.py

@@ -0,0 +1,2 @@
+# GENERATED BY KOMAND SDK - DO NOT EDIT
+from .action import Decode

unit_test/plugin_examples/good_plugin_with_cache/komand_base64/actions/decode/action.py

@@ -0,0 +1,29 @@
+import insightconnect_plugin_runtime
+from .schema import DecodeInput, DecodeOutput, Input, Output, Component
+from insightconnect_plugin_runtime.exceptions import PluginException
+import base64
+
+
+class Decode(insightconnect_plugin_runtime.Action):
+
+    def __init__(self):
+        super(self.__class__, self).__init__(
+                name='decode',
+                description=Component.DESCRIPTION,
+                input=DecodeInput(),
+                output=DecodeOutput())
+
+    def run(self, params={}):
+        try:
+            data = params.get(Input.BASE64)
+            errors = params.get(Input.ERRORS)
+            result = base64.standard_b64decode(data)
+            if errors in ["replace", "ignore"]:
+                return {Output.DATA: result.decode('utf-8', errors=errors)}
+            else:
+                return {Output.DATA: result.decode('utf-8')}
+        except Exception as e:
+            self.logger.error("An error has occurred while decoding ", e)
+            raise PluginException(cause="Internal error",
+                                  assistance='An error has occurred while decoding ',
+                                  data=e)