Add record framework (#225)

* Add CLI change in that use the helper module

* Added ability to block all apps

Some Windows apps will block, some will not. For example, Visual Code
is launched with code.cmd, which launches code.exe. Since code.cmd
ends quickly, we don't get a chance to edit. So we have blocking
that checks to see if the process name is actually running.

MacOS UI apps have a different kind of blocking. The application
needs to end.

Also fixed the problem with the record type json not being included
in the sdist.

* Add github actions and handle parsing better

* Add test for config command

Add test for config and fixed a few problem. The color --enable/disabled did
not work.

* Allow user to recheck if record data file

If the editor they are using is nonblocking, the CLI
will complain about template markers being found before the user
saves.

This allows the user to finished editing and then typing 'r' to
recheck the file.

* Handle output and exceptions better

* Add missing module to requirements.txt and setup.py

* Tests are failing in GitHub action. Print the command results.

* Fixed unit test and error message for config commands

The unit test was failing because no keeper.ini file existed. Use the Mock
config and the Export module too make a fake keeper.ini.

Also fixed some of the error when a command fails.

.github/workflows/publish.pypi.helper.yml

@@ -0,0 +1,45 @@
+name: Publish to PyPI (KSM SDK Helper)
+on:
+  workflow_dispatch:
+
+jobs:
+  publish-pypi:
+    name: Publish KSM SDK Helper to PyPI
+    environment: prod
+    runs-on: ubuntu-latest
+    timeout-minutes: 10 # To keep builds from running too long
+
+    defaults:
+      run:
+        working-directory: ./sdk/python/helper
+
+    steps:
+      - name: Get the source code
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.9
+
+      - name: Retrieve secrets from KSM
+        id: ksmsecrets
+        uses: Keeper-Security/ksm-action@master
+        with:
+          keeper-secret-config: ${{ secrets.KSM_PYPI_PUBLISHER_PYPI_SDK_CONFIG }}
+          secrets: |
+            -aBWi3-yU_qvatNh0Eaqew/field/password > PYPI_API_TOKEN
+
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade setuptools pip wheel twine
+          python3 -m pip install -r requirements.txt
+
+      - name: Build and Publish
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ steps.ksmsecrets.outputs.PYPI_API_TOKEN }}
+        run: |
+          python3 setup.py build
+          python3 setup.py sdist
+          python3 -m twine upload --verbose dist/*

.github/workflows/test.cli.yml

@@ -32,6 +32,20 @@ jobs:
         run: |
           python3 setup.py build install
 
+      ########## KSM Python Helper (from source)
+
+      - name: Install SDK Helper dependencies
+        working-directory: ./sdk/python/helper
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install -r requirements.txt
+          python3 -m pip install -e .
+
+      - name: Install SDK Helper for integrations
+        working-directory: ./sdk/python/helper
+        run: |
+          python3 setup.py build install
+
       ########## CLI
 
       - name: Install CLI dependencies

.github/workflows/test.python.helper.yml

@@ -0,0 +1,46 @@
+name: Test-Python-Helper
+
+on:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test-cli:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      ########## KSM Python SDK (from source)
+
+      - name: Install SDK dependencies
+        working-directory: ./sdk/python/core
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install -r requirements.txt
+          python3 -m pip install -e .
+
+      - name: Install SDK for integrations
+        working-directory: ./sdk/python/core
+        run: |
+          python3 setup.py build install
+
+      ########## PYTHON HELPER
+
+      - name: Install CLI dependencies
+        working-directory: ./sdk/python/helper
+        run: |
+          python3 -m pip install -r requirements.txt
+          python3 -m pip install pytest pytest-cov
+
+      - name: Run CLI tests
+        working-directory: ./sdk/python/helper
+        run: |
+          PYTHONPATH=$PWD pytest

integration/keeper_secrets_manager_cli/keeper_secrets_manager_cli/__init__.py

@@ -14,6 +14,8 @@
 from keeper_secrets_manager_core.core import KSMCache
 from keeper_secrets_manager_core.storage import InMemoryKeyValueStorage
 from keeper_secrets_manager_core.configkeys import ConfigKeys
+from keeper_secrets_manager_cli.common import find_ksm_path
+from keeper_secrets_manager_helper.record_type import RecordType
 from distutils.util import strtobool
 from .exception import KsmCliException
 from .profile import Profile
@@ -27,13 +29,22 @@ class KeeperCli:
     def get_client(**kwargs):
         return SecretsManager(**kwargs)
 
-    def __init__(self, ini_file=None, profile_name=None, output=None, use_color=None, use_cache=None):
+    def __init__(self, ini_file=None, profile_name=None, output=None, use_color=None, use_cache=None,
+                 record_type_dir=None, editor=None, editor_use_blocking=False, editor_process_name=None):
 
         self.profile = Profile(cli=self, ini_file=ini_file)
         self._client = None
 
         self.log_level = os.environ.get("KSM_DEBUG", None)
         self.use_color = use_color
+        self.record_type_dir = record_type_dir
+
+        # The editor to launch ... however this might be a bat or cmd file, not the real application
+        self.editor = editor
+        # Some application don't block. To enabling blocking the CLI, set this to True
+        self.editor_use_blocking = editor_use_blocking
+        # Blocking might be waiting until a process in the task list goes away. This is that process.
+        self.editor_process_name = editor_process_name
 
         self.use_cache = use_cache
 
@@ -56,6 +67,7 @@ def __init__(self, ini_file=None, profile_name=None, output=None, use_color=None
             config_storage.set(ConfigKeys.KEY_PRIVATE_KEY, self.config.get("privateKey"))
             config_storage.set(ConfigKeys.KEY_APP_KEY, self.config.get("appKey"))
             config_storage.set(ConfigKeys.KEY_HOSTNAME, self.config.get("hostname"))
+            config_storage.set(ConfigKeys.KEY_OWNER_PUBLIC_KEY, self.config.get("appOwnerPublicKey"))
 
             common_profile = self.profile.get_profile_config(Profile.config_profile)
 
@@ -74,6 +86,23 @@ def __init__(self, ini_file=None, profile_name=None, output=None, use_color=None
             # By default, use colors.
             if self.use_color is None:
                 self.use_color = bool(strtobool(common_profile.get(Profile.color_key, str(True))))
+
+            if self.record_type_dir is None:
+                self.record_type_dir = common_profile.get(Profile.record_type_dir_key, None)
+                if self.record_type_dir is None:
+                    self.record_type_dir = find_ksm_path("record_type", is_file=False)
+
+            # If the have a directory where record type schema files may exists, attempt to load
+            # them.
+            if self.record_type_dir is not None and os.path.exists(self.record_type_dir) is True:
+                RecordType.find_and_load_record_type_schema_files(self.record_type_dir)
+
+            # Get the editor to use for visual editing a record
+            if self.editor is None:
+                self.editor = common_profile.get(Profile.editor_key, None)
+            self.editor_use_blocking = bool(strtobool(common_profile.get(Profile.editor_use_blocking_key,
+                                                                         str(editor_use_blocking))))
+            self.editor_process_name = common_profile.get(Profile.editor_process_name_key, editor_process_name)
         else:
             # Set the log level. We don't have the client to set the level, so set it here.
             if use_color is None: