Release

examples/one_time_share.py

@@ -0,0 +1,31 @@
+import argparse
+import logging
+import sys
+
+from keepercommander import api
+from keepercommander.__main__ import get_params_from_config
+from keepercommander.commands import register
+
+parser = argparse.ArgumentParser(description='Create one time share URL')
+parser.add_argument('--name', dest='share_name', action='store', help='one-time share URL name')
+parser.add_argument('-e', '--expire', dest='expire', action='store', metavar='<NUMBER>[(m)inutes|(h)ours|(d)ays]',
+                                          help='Time period record share URL is valid.')
+parser.add_argument('record', nargs='?', type=str, action='store', help='record path or UID')
+
+opts, flags = parser.parse_known_args(sys.argv[1:])
+
+# Load Keeper configuration file
+my_params = get_params_from_config('')
+
+# Login to Keeper
+api.login(my_params)
+if not my_params.session_token:
+    exit(1)
+
+# Load vault
+api.sync_down(my_params)
+
+# Create one time share command
+cmd = register.OneTimeShareCreateCommand()
+url = cmd.execute(my_params, record=opts.record, share_name=opts.share_name, expire=opts.expire)
+print(url)

keepercommander/__init__.py

@@ -10,4 +10,4 @@
 # Contact: ops@keepersecurity.com
 #
 
-__version__ = '16.10.4'
+__version__ = '16.10.5'

keepercommander/api.py

@@ -693,7 +693,6 @@ def execute_batch(params, requests):
     throttle_delay = 10
     chunk_size = 999
     queue = requests.copy()
-    unthrottled = 0
     delay_next_batch = False
 
     while len(queue) > 0:
@@ -707,23 +706,19 @@ def execute_batch(params, requests):
         try:
             if delay_next_batch:
                 time.sleep(throttle_delay)
-                unthrottled = 0
             rs = communicate(params, rq)
             if 'results' in rs:
                 results = rs['results']  # type: list
                 if len(results) > 0:
-                    throttled = [r for r in results if r['result'] != 'success' and r['result_code'] == 'throttled']
+                    error_rs = results[-1]
+                    throttled = error_rs.get('result') != 'success' and error_rs.get('result_code') == 'throttled'
                     if throttled:
-                        throttled_rs = next(iter(throttled))
-                        throttled_idx = results.index(throttled_rs)
-                        queue = chunk[throttled_idx:] + queue
-                        responses.extend(results[:throttled_idx])
-                        unthrottled += throttled_idx
-                        chunk_size = unthrottled
                         delay_next_batch = True
-                    else:
-                        responses.extend(results)
-                        unthrottled += len(results)
+                        results.pop()
+                responses.extend(results)
+
+                if len(results) < len(chunk):
+                    queue = chunk[len(results):] + queue
         except Exception as e:
             logging.error(e)
 
@@ -1276,11 +1271,11 @@ def need_share_info(uid):
         return result
 
 
-def query_enterprise(params, force=False):
+def query_enterprise(params, force=False, tree_key=None):
     try:
         if force is True and params.enterprise:
             params.enterprise = None
-        qe(params)
+        qe(params, tree_key=tree_key)
     except Exception as e:
         share_account_by = params.get_share_account_timestamp()
         share_account_expired = share_account_by and datetime.today() > share_account_by
@@ -1314,9 +1309,10 @@ def login_and_get_mc_params_login_v3(params: KeeperParams, mc_id):
 
     mc_params.session_token = loginv3.CommonHelperMethods.bytes_to_url_safe_str(resp.encryptedSessionToken)
     mc_params.msp_tree_key = params.enterprise['unencrypted_tree_key']
+    tree_key = crypto.decrypt_aes_v2(utils.base64_url_decode(resp.encryptedTreeKey), mc_params.msp_tree_key)
 
     sync_down(mc_params)
-    query_enterprise(mc_params, True)
+    query_enterprise(mc_params, True, tree_key=tree_key)
 
     return mc_params
 

keepercommander/commands/discoveryrotation.py

@@ -1564,7 +1564,7 @@ def execute(self, params, **kwargs):
 
 ############################################## TUNNELING ###############################################################
 class PAMTunnelListCommand(Command):
-    pam_cmd_parser = argparse.ArgumentParser(prog='dr-tunnel-list-command')
+    pam_cmd_parser = argparse.ArgumentParser(prog='pam tunnel list')
 
     def get_parser(self):
         return PAMTunnelListCommand.pam_cmd_parser
@@ -1648,7 +1648,7 @@ def clean_up_tunnel(params, convo_id):
 
 
 class PAMTunnelStopCommand(Command):
-    pam_cmd_parser = argparse.ArgumentParser(prog='dr-tunnel-stop-command')
+    pam_cmd_parser = argparse.ArgumentParser(prog='pam tunnel stop')
     pam_cmd_parser.add_argument('uid', type=str, action='store', help='The Tunnel UID')
 
     def get_parser(self):
@@ -1668,7 +1668,7 @@ def execute(self, params, **kwargs):
 
 
 class PAMTunnelTailCommand(Command):
-    pam_cmd_parser = argparse.ArgumentParser(prog='dr-tunnel-tail-command')
+    pam_cmd_parser = argparse.ArgumentParser(prog='pam tunnel tail')
     pam_cmd_parser.add_argument('uid', type=str, action='store', help='The Tunnel UID')
 
     def get_parser(self):
@@ -1737,18 +1737,22 @@ def retrieve_gateway_public_key(gateway_uid, params, api, utils) -> bytes:
 
 
 class PAMTunnelEnableCommand(Command):
-    pam_cmd_parser = argparse.ArgumentParser(prog='dr-tunnel-enable-command')
+    pam_cmd_parser = argparse.ArgumentParser(prog='pam tunnel enable')
     pam_cmd_parser.add_argument('uid', type=str, action='store', help='The Record UID of the PAM '
                                                                       'resource record with network information to use '
                                                                       'for tunneling')
+    pam_cmd_parser.add_argument('--configuration', '-c', required=False, dest='config', action='store',
+                                help='The PAM Configuration UID to use for tunneling. '
+                                     'Use command `pam config list` to view available PAM Configurations.')
 
     def get_parser(self):
         return PAMTunnelEnableCommand.pam_cmd_parser
 
     def execute(self, params, **kwargs):
         record_uid = kwargs.get('uid')
+        config_uid = kwargs.get('config')
         if not record_uid:
-            raise CommandError('tunnel Enable', '"record" argument is required')
+            raise CommandError('tunnel Enable', '"record UID" argument is required')
         dirty = False
 
         record = vault.KeeperRecord.load(params, record_uid)
@@ -1757,15 +1761,42 @@ def execute(self, params, **kwargs):
             print(f"{bcolors.FAIL}Record {record_uid} not found.{bcolors.ENDC}")
             return
 
+        record_type = record.record_type
+        if record_type not in "pamMachine pamDatabase pamDirectory".split():
+            print(f"{bcolors.FAIL}This record's type is not supported for tunnels. "
+                  f"Tunnels are only supported on Pam Machine, Pam Database, and Pam Directory records{bcolors.ENDC}")
+            return
+
+        if config_uid:
+            configuration = vault.KeeperRecord.load(params, config_uid)
+            if not isinstance(configuration, vault.TypedRecord):
+                print(f"{bcolors.FAIL}Configuration {config_uid} not found.{bcolors.ENDC}")
+                return
+            if configuration.record_type != 'pamNetworkConfiguration':
+                print(f"{bcolors.FAIL}The record {config_uid} is not a Pam Configuration.{bcolors.ENDC}")
+                return
+
         pam_settings = record.get_typed_field('pamSettings')
         if not pam_settings:
-            pam_settings = vault.TypedField.new_field('pamSettings',
-                                                      {"portForward": {"enabled": True}}, "")
+            pre_settings = {"portForward": {"enabled": True}}
+            if config_uid:
+                pre_settings["configUid"] = config_uid
+            pam_settings = vault.TypedField.new_field('pamSettings', pre_settings, "")
             record.custom.append(pam_settings)
+            dirty = True
         else:
+            if config_uid:
+                if pam_settings.value[0].get('configUid') != config_uid:
+                    pam_settings.value[0]['configUid'] = config_uid
+                    dirty = True
             if not pam_settings.value[0]['portForward']['enabled']:
                 pam_settings.value[0]['portForward']['enabled'] = True
                 dirty = True
+        if not pam_settings.value[0].get('configUid'):
+            print(f"{bcolors.FAIL}No PAM Configuration UID set. This must be set for tunneling to work. "
+                  f"This can be done by running 'pam tunnel enable {record_uid} --config [ConfigUID]' "
+                  f"The ConfigUID can be found by running 'pam config list'{bcolors.ENDC}")
+            return
 
         client_private_key = record.get_typed_field('trafficEncryptionKey')
         if not client_private_key:
@@ -1791,10 +1822,15 @@ def execute(self, params, **kwargs):
         if dirty:
             record_management.update_record(params, record)
             api.sync_down(params)
+        if pam_settings.value[0].get('configUid'):
+            print(f"{bcolors.OKGREEN}Tunneling enabled for {record_uid} using configuration "
+                  f"{pam_settings.value[0].get('configUid')} {bcolors.ENDC}")
+        else:
+            print(f"{bcolors.OKGREEN}Tunneling enabled for {record_uid}{bcolors.ENDC}")
 
 
 class PAMTunnelDisableCommand(Command):
-    pam_cmd_parser = argparse.ArgumentParser(prog='dr-tunnel-disable-command')
+    pam_cmd_parser = argparse.ArgumentParser(prog='pam tunnel disable')
     pam_cmd_parser.add_argument('uid', type=str, action='store', help='The Record UID of the PAM '
                                                                       'resource record with network information to use '
                                                                       'for tunneling')
@@ -1820,15 +1856,14 @@ def execute(self, params, **kwargs):
                 pam_settings.value[0]['portForward']['enabled'] = False
                 record_management.update_record(params, record)
                 api.sync_down(params)
+        print(f"{bcolors.OKGREEN}Tunneling disabled for {record_uid}{bcolors.ENDC}")
 
 
 class PAMTunnelStartCommand(Command):
-    pam_cmd_parser = argparse.ArgumentParser(prog='dr-port-forward-command')
-    pam_cmd_parser.add_argument('--gateway', '-g', required=True, dest='gateway', action='store',
-                                help='Used to list all tunnels for the given Gateway UID')
-    pam_cmd_parser.add_argument('--record', '-r', required=True, dest='record_uid', action='store',
-                                help='The Record UID of the PAM resource record with network information to use for '
-                                     'tunneling')
+    pam_cmd_parser = argparse.ArgumentParser(prog='pam tunnel start')
+    pam_cmd_parser.add_argument('uid', type=str, action='store', help='The Record UID of the PAM resource '
+                                                                      'record with network information to use for '
+                                                                      'tunneling')
     pam_cmd_parser.add_argument('--host', '-o', required=False, dest='host', action='store',
                                 default="127.0.0.1",
                                 help='The address on which the server will be accepting connections. It could be an '
@@ -2028,10 +2063,9 @@ def execute(self, params, **kwargs):
                   f"You are using {major_version}.{minor_version}.{micro_version}.{bcolors.ENDC}")
             return
 
-        record_uid = kwargs.get('record_uid')
+        record_uid = kwargs.get('uid')
         convo_id = GatewayAction.generate_conversation_id()
         params.tunnel_threads[convo_id] = {}
-        gateway_uid = kwargs.get('gateway')
         host = kwargs.get('host')
         port = kwargs.get('port')
         if port is not None and port > 0:
@@ -2046,12 +2080,6 @@ def execute(self, params, **kwargs):
                 print(f"{bcolors.FAIL}Could not find open port to use for tunnel{bcolors.ENDC}")
                 return
 
-        gateway_public_key_bytes = retrieve_gateway_public_key(gateway_uid, params, api, utils)
-
-        if not gateway_public_key_bytes:
-            print(f"{bcolors.FAIL}Could not retrieve public key for gateway {gateway_uid}{bcolors.ENDC}")
-            return
-
         api.sync_down(params)
         record = vault.KeeperRecord.load(params, record_uid)
         if not isinstance(record, vault.TypedRecord):
@@ -2060,14 +2088,17 @@ def execute(self, params, **kwargs):
 
         pam_settings = record.get_typed_field('pamSettings')
         if not pam_settings:
-            print(f"{bcolors.FAIL}PAM Settings not enabled for record {record_uid}.{bcolors.ENDC}")
+            print(f"{bcolors.FAIL}PAM Settings not enabled for record {record_uid}'.{bcolors.ENDC}")
+            print(f"{bcolors.WARNING}This is done by running 'pam tunnel enable {record_uid} "
+                  f"--config [ConfigUID]' The ConfigUID can be found by running 'pam config list'{bcolors.ENDC}.")
             return
 
         try:
-            dag_info = pam_settings.value[0]
-            enabled_port_forward = dag_info.get("portForward", {}).get("enabled", False)
+            pam_info = pam_settings.value[0]
+            enabled_port_forward = pam_info.get("portForward", {}).get("enabled", False)
             if not enabled_port_forward:
-                print(f"{bcolors.FAIL}PAM Settings not enabled for record {record_uid}.{bcolors.ENDC}")
+                print(f"{bcolors.FAIL}PAM Settings not enabled for record {record_uid}. "
+                      f"{bcolors.WARNING}This is done by running 'pam tunnel enable {record_uid}'.{bcolors.ENDC}")
                 return
         except Exception as e:
             print(f"{bcolors.FAIL}Error parsing PAM Settings for record {record_uid}: {e}{bcolors.ENDC}")
@@ -2080,6 +2111,39 @@ def execute(self, params, **kwargs):
 
         client_private_key_value = client_private_key.get_default_value(str)
 
+        configuration_uid = pam_info.get("configUid", None)
+        if not configuration_uid:
+            print(f"{bcolors.FAIL}Configuration UID not found for record {record_uid}.{bcolors.ENDC}")
+            return
+        configuration = vault.KeeperRecord.load(params, configuration_uid)
+        if not isinstance(configuration, vault.TypedRecord):
+            print(f"{bcolors.FAIL}Configuration {configuration_uid} not found.{bcolors.ENDC}")
+            return
+
+        pam_resources = configuration.get_typed_field('pamResources')
+        if not pam_resources:
+            print(f"{bcolors.FAIL}PAM Resources not found for configuration {configuration_uid}.{bcolors.ENDC}")
+            return
+        if len(pam_resources.value) == 0:
+            print(f"{bcolors.FAIL}PAM Resources not found for configuration {configuration_uid}.{bcolors.ENDC}")
+            return
+        gateway_uid = ''
+        try:
+            gateway_uid = pam_resources.value[0].get("controllerUid", '')
+        except Exception as e:
+            print(f"{bcolors.FAIL}Error parsing PAM Resources for configuration {configuration_uid}: {e}{bcolors.ENDC}")
+            CommandError('Tunnel Start', f"{e}")
+
+        if not gateway_uid:
+            print(f"{bcolors.FAIL}Gateway UID not found for configuration {configuration_uid}.{bcolors.ENDC}")
+            return
+
+        gateway_public_key_bytes = retrieve_gateway_public_key(gateway_uid, params, api, utils)
+
+        if not gateway_public_key_bytes:
+            print(f"{bcolors.FAIL}Could not retrieve public key for gateway {gateway_uid}{bcolors.ENDC}")
+            return
+
         t = threading.Thread(target=self.pre_connect, args=(params, record_uid, convo_id, gateway_uid, host, port,
                                                             gateway_public_key_bytes, client_private_key_value)
                              )

keepercommander/commands/folder.py

@@ -37,7 +37,8 @@
 from ..proto import folder_pb2, record_pb2
 from ..record import Record
 from ..recordv3 import RecordV3
-from ..subfolder import BaseFolderNode, try_resolve_path, find_folders, SharedFolderNode, get_contained_record_uids
+from ..subfolder import BaseFolderNode, try_resolve_path, find_folders, SharedFolderNode, get_contained_record_uids, \
+    get_contained_folder_uids
 
 
 def register_commands(commands):
@@ -70,6 +71,8 @@ def register_command_info(aliases, command_info):
 ls_parser.add_argument('-s', '--short', dest='short', action='store_true',
                        help='Do not display record details. (Not used)')
 ls_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='verbose output')
+recursive_help = 'list all folders/records in subfolders'
+ls_parser.add_argument('-R', '--recursive', dest='recursive', action='store_true', help=recursive_help)
 ls_parser.add_argument('pattern', nargs='?', type=str, action='store', help='search pattern')
 ls_parser.error = raise_parse_exception
 ls_parser.exit = suppress_exit
@@ -216,16 +219,20 @@ def execute(self, params, **kwargs):
         folders = []    # type: List[BaseFolderNode]
         records = []    # type: List[vault.KeeperRecord]
 
+        recursive_search = kwargs.get('recursive')
+        folder_uid = folder.uid or ''
         if show_folders:
-            for uid in folder.subfolders:
+            sub_folder_uids = get_contained_folder_uids(params, folder_uid, not recursive_search)
+            for uid in sub_folder_uids:
                 f = params.folder_cache[uid]
                 if any(filter(lambda x: regex(x) is not None, FolderListCommand.folder_match_strings(f))) if regex is not None else True:
                     folders.append(f)
 
         if show_records and params.record_cache:
-            folder_uid = folder.uid or ''
             if folder_uid in params.subfolder_record_cache:
-                for uid in params.subfolder_record_cache[folder_uid]:
+                record_uids_by_folder = get_contained_record_uids(params, folder_uid, not recursive_search)
+                record_uids = {rec_uid for recs in record_uids_by_folder.values() for rec_uid in recs}
+                for uid in record_uids:
                     if uid not in params.record_cache:
                         continue
                     rec = params.record_cache[uid]
@@ -1750,7 +1757,7 @@ def tree_node(node):
         rec_nodes = []
         if show_records and isinstance(node, BaseFolderNode):
             node_uid = '' if node.type == '/' else node.uid
-            rec_uids = get_contained_record_uids(params, node_uid).get(node_uid)
+            rec_uids = {rec for recs in get_contained_record_uids(params, node_uid).values() for rec in recs}
             records = [api.get_record(params, rec_uid) for rec_uid in rec_uids]
             records = [r for r in records if isinstance(r, Record)]
             rec_nodes.extend(records)

keepercommander/commands/msp.py

@@ -1315,11 +1315,11 @@ def execute(self, params, **kwargs):
                 for enforcement in src_enforcements:
                     src_value = src_enforcements[enforcement]
                     if enforcement in dst_enforcements:
-                        command = 'role_enforcement_update'
                         dst_value = dst_enforcements[enforcement]
-                        if src_value != dst_value:
-                            command = 'role_enforcement_update'
                         dst_enforcements.pop(enforcement)
+                        if src_value == dst_value:
+                            continue
+                        command = 'role_enforcement_update'
                     else:
                         command = 'role_enforcement_add'
                     if command: