Release KSM CLI v1.1.0 (#475)

* KSM CLI: bump version to 1.1.0 and updated change log

* KSM CLI: Added new options to allow using aws managed secrets as config storage (#472)

integration/keeper_secrets_manager_ansible/requirements.txt

@@ -1,4 +1,4 @@
-ansible
+ansible<8.0.0
 importlib_metadata
 keeper-secrets-manager-core>=16.4.1
 keeper-secrets-manager-helper>=1.0.4

integration/keeper_secrets_manager_ansible/setup.py

@@ -12,7 +12,7 @@
     'keeper-secrets-manager-core>=16.4.1',
     'keeper-secrets-manager-helper>=1.0.4',
     'importlib_metadata',
-    'ansible'
+    'ansible<8.0.0'
 ]
 
 setup(

integration/keeper_secrets_manager_ansible/tests/keeper_copy_test.py

@@ -72,7 +72,7 @@ def _common(self):
             r, out, err = a.run()
             result = r[0]["localhost"]
             self.assertEqual(result["ok"], 4, "4 things didn't happen")
-            self.assertEqual(result["failures"], 0, "failures was n ot 0")
+            self.assertEqual(result["failures"], 0, "failures was not 0")
             self.assertEqual(result["changed"], 3, "3 things didn't change")
             ls = os.listdir(temp_dir)
             self.assertTrue("password" in ls, "did not find file password")

integration/keeper_secrets_manager_cli/README.md

@@ -6,6 +6,9 @@ For more information see our official documentation page https://docs.keeper.io/
 
 # Change History
 
+## 1.1.0
+* KSM-395 - New feature to load configurations from AWS Secrets Manager
+
 ## 1.0.17
 * KSM-392 - Ability to update fields where the label is a blank string (`""`)
 * Pinned KSM Core version to 16.5.1

integration/keeper_secrets_manager_cli/keeper_secrets_manager_cli/__main__.py

@@ -51,6 +51,7 @@ class AliasedGroup(HelpColorsGroup):
         "export",
         "import",
         "init",
+        "setup",
         "sync",
         "secret",
         "totp",
@@ -297,7 +298,6 @@ def cli(ctx, ini_file, profile_name, output, color, cache, log_level):
 
 
 # PROFILE GROUP
-
 @click.group(
     name='profile',
     cls=AliasedGroup,
@@ -343,6 +343,97 @@ def profile_init_command(ctx, token, hostname, ini_file, profile_name, token_arg
     )
 
 
+@click.command(
+    name='setup',
+    cls=HelpColorsCommand,
+    help_options_color='blue'
+)
+@click.option('--type', '-t', required=True, type=click.Choice(['aws']), help="The type of the remote storage: aws/azure/gcp - currently only aws is supported.")
+@click.option('--secret', '-s', required=False, type=str, default='ksm-config', help="Secret's name or full URI in Secrets Manager.")
+@click.option('--credentials', '-c', required=False, type=click.Choice(['ec2instance', 'profile', 'keys']), default='ec2instance', help="The type of the credentials for the remote storage. Default value is ec2instance")
+@click.option('--credentials-profile', '-n', required=False, type=str, help="Profile name from local machine config.")
+@click.option('--aws-access-key-id', required=False, type=str, help="AWS Access Key ID.")
+@click.option('--aws-secret-access-key', required=False, type=str, help="AWS Secret Access Key.")
+@click.option('--region', required=False, type=str, help="AWS region.")
+@click.option('--fallback', '-f', is_flag=False, help='If credentials fail then fallback to default profile on the machine.')
+@click.option('--ini-file', type=str, help="INI config file to create.")
+@click.option('--profile-name', '-p', type=str, help='Config profile to create.')
+@click.pass_context
+def profile_setup_command(ctx, type, secret, credentials,
+                          credentials_profile,
+                          aws_access_key_id, aws_secret_access_key, region,
+                          fallback, ini_file, profile_name):
+    """Setup a profile to load config from remote storage"""
+
+    # Since the top level options are available for all commands,
+    # it might be confusing the setup command
+    if ctx.obj["ini_file"] is not None and ini_file is not None:
+        print("NOTE: The INI file config was set on the top level command and"
+              " also set on the setup sub-command. The top level command"
+              " parameter will be ignored for the setup sub-command.",
+              file=sys.stderr)
+
+    if type == 'aws':
+        if not secret:
+            secret = 'ksm-config'
+        if not credentials:
+            credentials = 'ec2instance'
+
+        # credentials options
+        # ec2instance: doesn't require additional options
+        # profile: accepts only --credentials-profile=NAME
+        # keys: requires both keys and region
+        if credentials == 'ec2instance':
+            if (credentials_profile or
+                    aws_access_key_id or aws_secret_access_key or region):
+                raise click.ClickException(
+                    "Unexpected options for --credentials=ec2instance "
+                    "which doesn't require additional parameters. Please "
+                    "do not pass other settings (profile/key/region)")
+            Profile.from_aws_ec2instance(
+                secret=secret,
+                fallback=fallback,
+                ini_file=ini_file,
+                profile_name=profile_name,
+                launched_from_app=global_config.launched_from_app)
+        elif credentials == 'profile':
+            credentials_profile = credentials_profile or ""
+            # accepts only one optional parameter -cp|credentials-profile=NAME
+            if aws_access_key_id or aws_secret_access_key or region:
+                raise click.ClickException(
+                    "Unexpected options for --credentials=profile "
+                    "which accepts only one optional parameter "
+                    "--credentials-profile=NAME "
+                    "Please do not pass any keys (key/region)")
+            Profile.from_aws_profile(
+                secret=secret,
+                fallback=fallback,
+                aws_profile=credentials_profile,
+                ini_file=ini_file,
+                profile_name=profile_name,
+                launched_from_app=global_config.launched_from_app)
+        elif credentials == 'keys':
+            if credentials_profile:
+                raise click.ClickException(
+                    f"With --credentials-profile={credentials_profile} "
+                    "must specify option --credentials=profile")
+            # requires: aws-access-key-id, aws-secret-access-key, region
+            if not (aws_access_key_id and aws_secret_access_key and region):
+                raise click.ClickException(
+                    "Missing options for --credentials=keys "
+                    "which requires both keys and region to be set with "
+                    "--aws-access-key-id, --aws-secret-access-key, --region")
+            Profile.from_aws_custom(
+                secret=secret,
+                fallback=fallback,
+                aws_access_key_id=aws_access_key_id,
+                aws_secret_access_key=aws_secret_access_key,
+                region=region,
+                ini_file=ini_file,
+                profile_name=profile_name,
+                launched_from_app=global_config.launched_from_app)
+
+
 @click.command(
     name='list',
     cls=HelpColorsCommand,
@@ -411,15 +502,14 @@ def profile_import_command(ctx, output_file, config_base64):
 
 
 profile_command.add_command(profile_init_command)
+profile_command.add_command(profile_setup_command)
 profile_command.add_command(profile_list_command)
 profile_command.add_command(profile_active_command)
 profile_command.add_command(profile_export_command)
 profile_command.add_command(profile_import_command)
 
 
 # SECRET GROUP
-
-
 @click.group(
     name='secret',
     cls=AliasedGroup,
@@ -840,7 +930,6 @@ def exec_command(ctx, capture_output, inline, cmd):
 def config_command(ctx):
     """Configure the command line tool"""
     ctx.obj["profile"] = Profile(cli=ctx.obj["cli"], config=global_config)
-    pass
 
 
 @click.command(
@@ -1071,7 +1160,6 @@ def quit_command():
 
 
 # SYNC COMMAND
-
 @click.command(
     name='sync',
     cls=HelpColorsCommand,

integration/keeper_secrets_manager_cli/keeper_secrets_manager_cli/config.py

@@ -7,28 +7,32 @@
 #
 # Keeper Secrets Manager
 # Copyright 2021 Keeper Security Inc.
-# Contact: ops@keepersecurity.com
+# Contact: sm@keepersecurity.com
 #
 
-from keeper_secrets_manager_cli.common import find_ksm_path
-from keeper_secrets_manager_cli.exception import KsmCliException
-from keeper_secrets_manager_core.utils import set_config_mode, check_config_mode
-from keeper_secrets_manager_core.keeper_globals import logger_name
-import logging
+import base64
 import colorama
 import configparser
+import json
+import logging
 import platform
 import os
-import base64
-import json
+
+from keeper_secrets_manager_core.configkeys import ConfigKeys
+from keeper_secrets_manager_core.keeper_globals import logger_name
+from keeper_secrets_manager_core.storage import InMemoryKeyValueStorage
+from keeper_secrets_manager_core.utils import set_config_mode, check_config_mode
+from keeper_secrets_manager_cli.common import find_ksm_path
+from keeper_secrets_manager_cli.exception import KsmCliException
 
 
 class Config:
 
     """
     Provide a structure representation of the keeper.ini.
 
-    Instead of using a bunch of dictionaries, use objects. Then use the attributes to hold data.
+    Instead of using a bunch of dictionaries, use objects.
+    Then use the attributes to hold data.
     """
 
     default_ini_file = os.environ.get("KSM_INI_FILE", "keeper.ini")
@@ -76,8 +80,8 @@ def create_from_json(json_config):
     def get_default_ini_file(launched_from_app=False):
         working_directory = os.getcwd()
 
-        # If launched from an application, the current working directory might not be writeable. Use
-        # the user's "HOME" directory.
+        # If launched from an application, the current working directory
+        # might not be writeable. Use the user's "HOME" directory.
         if launched_from_app is True:
             if Config.is_windows() is True:
                 working_directory = os.environ["USERPROFILE"]
@@ -108,7 +112,7 @@ def get_profile(self, name):
 
     def set_profile_using_base64(self, profile_name, base64_config):
 
-        # If the base64_config has already been decoded, the no need to
+        # If the base64_config has already been decoded, then no need to
         # base64 decode.
         if base64_config.strip().startswith("{") is True:
             json_config = base64_config
@@ -147,18 +151,65 @@ def load(self):
                     if profile_name == Config.CONFIG_KEY:
                         continue
 
-                    self._profiles[profile_name] = ConfigProfile(
-                        client_id=config[profile_name].get("clientid"),
-                        private_key=config[profile_name].get("privatekey"),
-                        app_key=config[profile_name].get("appkey"),
-                        hostname=config[profile_name].get("hostname"),
-                        app_owner_public_key=config[profile_name].get("appownerpublickey"),
-                        server_public_key_id=config[profile_name].get("serverpublickeyid"))
+                    self._profiles[profile_name] = self._load_config(config[profile_name])
         except PermissionError:
             raise PermissionError("Access denied to configuration file {}.".format(self.ini_file))
         except FileNotFoundError:
             raise PermissionError("Cannot find configuration file {}.".format(self.ini_file))
 
+    def _load_config(self, section: configparser.SectionProxy):
+        from keeper_secrets_manager_storage.storage_aws_secret import AwsConfigProvider
+
+        storage = section.get("storage", "")
+        if storage in ("", "internal"):
+            return ConfigProfile(
+                    client_id=section.get("clientid"),
+                    private_key=section.get("privatekey"),
+                    app_key=section.get("appkey"),
+                    hostname=section.get("hostname"),
+                    app_owner_public_key=section.get("appownerpublickey"),
+                    server_public_key_id=section.get("serverpublickeyid"))
+        elif storage == "aws":
+            cfg = ConfigProfile(storage=storage)
+            cfg.storage_config = {x: section.get(x) for x in section.keys() if x != "storage"}
+
+            provider = cfg.storage_config.get("provider", "") or "ec2instance"
+            secret = cfg.storage_config.get("secret", "") or "ksm-config"
+            fallback = cfg.storage_config.get("fallback", True) or True
+
+            awsp = AwsConfigProvider(secret)
+            if provider == "ec2instance":
+                awsp.from_ec2instance_config(secret, fallback)
+            elif provider == "profile":
+                profile = cfg.storage_config.get("profile", "") or ""
+                if profile:
+                    awsp.from_profile_config(secret, profile, fallback)
+                else:
+                    awsp.from_default_config(secret, fallback)
+            elif provider == "keys":
+                aws_access_key_id = cfg.storage_config.get("aws_access_key_id", "") or ""
+                aws_secret_access_key = cfg.storage_config.get("aws_secret_access_key", "") or ""
+                region = cfg.storage_config.get("region", "") or ""
+                awsp.from_custom_config(secret, aws_access_key_id, aws_secret_access_key, region, fallback)
+            else:
+                raise KsmCliException(f"Failed to load profile from AWS secret - unknown provider '{provider}'")
+
+            ksmcfg = awsp.read_config()
+            if not ksmcfg:
+                raise KsmCliException(f"Failed to load profile from AWS secret '{secret}'")
+
+            config_storage = InMemoryKeyValueStorage(ksmcfg)
+            cfg.client_id = config_storage.get(ConfigKeys.KEY_CLIENT_ID)
+            cfg.private_key = config_storage.get(ConfigKeys.KEY_PRIVATE_KEY)
+            cfg.app_key = config_storage.get(ConfigKeys.KEY_APP_KEY)
+            cfg.hostname = config_storage.get(ConfigKeys.KEY_HOSTNAME)
+            cfg.app_owner_public_key = config_storage.get(ConfigKeys.KEY_OWNER_PUBLIC_KEY)
+            cfg.server_public_key_id = config_storage.get(ConfigKeys.KEY_SERVER_PUBLIC_KEY_ID)
+
+            return cfg
+        else:
+            raise KsmCliException("Unknown profile storage '{storage}' - please update KSM CLI")
+
     def save(self):
         if self.has_config_file is True:
 
@@ -212,6 +263,9 @@ def to_dict(self):
 class ConfigProfile:
 
     def __init__(self, **kwargs):
+        # storage: internal|aws|azure|gcp - only internal is exportable
+        self.storage = kwargs.get("storage", "internal")
+        self.storage_config = kwargs.get("storage_config", {})
         self.client_id = kwargs.get("client_id")
         self.private_key = kwargs.get("private_key")
         self.app_key = kwargs.get("app_key")
@@ -220,11 +274,19 @@ def __init__(self, **kwargs):
         self.server_public_key_id = kwargs.get("server_public_key_id")
 
     def to_dict(self):
-        return {
+        result = {
+            # "storage": self.storage,  # removed for legacy compatibility
             "clientid": self.client_id,
             "privatekey": self.private_key,
             "appkey": self.app_key,
             "hostname": self.hostname,
             "appownerpublickey": self.app_owner_public_key,
             "serverpublickeyid": self.server_public_key_id
         }
+
+        if self.storage and self.storage != "internal":
+            result = {"storage": self.storage}
+            if self.storage_config:
+                result.update(self.storage_config)
+
+        return result

integration/keeper_secrets_manager_cli/keeper_secrets_manager_cli/export.py

@@ -28,6 +28,13 @@ class Export:
     """
 
     def __init__(self, config, file_format=None, plain=False):
+        # To prevent exposing cloud based secrets
+        # only configurations stored internally can be exported
+        if config.storage not in (None, "", "internal"):
+            raise KsmCliException(
+                "Only configurations stored internally can be exported. "
+                f" Current profile has storage={config.storage}")
+
         # If the JSON dictionary is passed in convert it to a Config
         if isinstance(config, dict) is True:
             config = Config.create_from_json(config).get_profile(Config.default_profile)