Add caching (based on PyFilesystem#28)

abretaud · Mar 10, 2021 · 2b65d21 · 2b65d21
1 parent 87704e2
commit 2b65d21
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 23 deletions.
diff --git a/setup.py b/setup.py
@@ -25,7 +25,8 @@
 REQUIREMENTS = [
     "fs>2.0",
     "webdavclient3",
-    "python-dateutil"
+    "python-dateutil",
+    "cachetools"
 ]
 
 setup(

diff --git a/webdavfs/webdavfs.py b/webdavfs/webdavfs.py
@@ -20,6 +20,7 @@
 from fs.iotools import line_iterator
 from fs.mode import Mode
 from fs.path import dirname
+from cachetools import TTLCache
 
 
 log = logging.getLogger(__name__)
@@ -156,7 +157,8 @@ class WebDAVFS(FS):
         'virtual': False,
     }
 
-    def __init__(self, url, login=None, password=None, root=None):
+    def __init__(self, url, login=None, password=None, root=None,
+                 cache_maxsize=10000, cache_ttl=60):
         self.url = url
         self.root = root
         super(WebDAVFS, self).__init__()
@@ -167,6 +169,8 @@ def __init__(self, url, login=None, password=None, root=None):
             'webdav_password': password,
             'root': self.root
         }
+        self.info_cache = TTLCache(maxsize=cache_maxsize,
+                                   ttl=cache_ttl)
         self.client = wc.Client(options)
 
     def _create_resource(self, path):
@@ -182,7 +186,8 @@ def _create_info_dict(info):
         info_dict = {
             'basic': {"is_dir": False},
             'details': {'type': int(ResourceType.file)},
-            'access': {}
+            'access': {},
+            'other': {}
         }
 
         if six.PY2:
@@ -208,7 +213,7 @@ def decode_datestring(s):
                 if key in ('modified', 'created', 'accessed'):
                     info_dict['details'][key] = decode(val) or None
                 else:
-                    info_dict['details'][key] = decode(val) 
+                    info_dict['details'][key] = decode(val)
             elif key in access:
                 info_dict['access'][key] = decode(val)
             else:
@@ -230,45 +235,65 @@ def exists(self, path):
 
     def getinfo(self, path, namespaces=None):
         _path = self.validatepath(path)
-        namespaces = namespaces or ()
 
         if _path in '/':
+            self.info_cache.clear()
             info_dict = {
                 "basic": {
                     "name": "",
                     "is_dir": True
                 },
                 "details": {
                     "type": ResourceType.directory
-                }
+                },
+                'access': {},
+                "other": {}
             }
 
-        else:
-            try:
-                info = self.client.info(_path.encode('utf-8'))
-                # displayname is optional
+        try:
+            namespaces = namespaces or ()
+            urn = wu.Urn(_path.encode('utf-8'))
+            path = self.client.get_full_path(urn)
+            if path in self.info_cache:
+                info = self.info_cache[path]
+            else:
+                response = self.client.execute_request(action='info',
+                                                       path=urn.quote())
+                info = wc.WebDavXmlUtils.parse_info_response(content=response.content, path=path, hostname=self.client.webdav.hostname)
                 if info['name'] is None:
                     info['name'] = _path.split("/")[-1]
-                info_dict = self._create_info_dict(info)
-                if self.client.is_dir(_path.encode('utf-8')):
-                    info_dict['basic']['is_dir'] = True
-                    info_dict['details']['type'] = ResourceType.directory
-            except we.RemoteResourceNotFound as exc:
-                raise errors.ResourceNotFound(path, exc=exc)
+                if wc.WebDavXmlUtils.parse_is_dir_response(content=response.content, path=path, hostname=self.client.webdav.hostname):
+                    info['isdir'] = True
+                    info['files'] = []
+                    for i in wc.WebDavXmlUtils.parse_get_list_info_response(response.content):
+                        if i['path'].rstrip('/') != path.rstrip('/'):
+                            if i['name'] is None:
+                                i['name'] = i['path'].split("/")[-1]
+                            self.info_cache[i['path']] = i
+                            filename = wu.Urn(i['path'], i['isdir']).filename()
+                            if six.PY2:
+                                filename = filename.decode('utf-8')
+                            filename = filename.rstrip('/')
+                            info['files'].append(filename)
+                self.info_cache[path] = info
+            info_dict = self._create_info_dict(info)
+            if info.get('isdir', False):
+                info_dict['basic']['is_dir'] = True
+                info_dict['details']['type'] = ResourceType.directory
+        except we.RemoteResourceNotFound as exc:
+            raise errors.ResourceNotFound(path, exc=exc)
 
         return Info(info_dict)
 
     def listdir(self, path):
-        _path = self.validatepath(path)
-
-        if not self.getinfo(_path).is_dir:
+        info = self.getinfo(path)
+        if not info.is_dir:
             raise errors.DirectoryExpected(path)
 
-        dir_list = self.client.list(_path.encode('utf-8'))
-        if six.PY2:
-            dir_list = map(operator.methodcaller('decode', 'utf-8'), dir_list)
+        for i in info.raw['other']['files']:
+            yield i
 
-        return list(map(operator.methodcaller('rstrip', '/'), dir_list))
+        return
 
     def makedir(self, path, permissions=None, recreate=False):
         _path = self.validatepath(path)