Implement asset list caching

bookmarks/actions.py

@@ -16,6 +16,7 @@
 from . import common
 from . import database
 from . import images
+from. import log
 
 
 def must_be_initialized(func):
@@ -883,6 +884,14 @@ def refresh(idx=None):
     """
     w = common.widget(idx=idx)
     model = w.model().sourceModel()
+
+    # Remove the asset list cache if we're forcing a refresh on the asset tab
+    if common.current_tab() == common.AssetTab:
+        cache = f'{common.active("root", path=True)}/{common.bookmark_cache_dir}/assets.cache'
+        if os.path.exists(cache):
+            print('Removing asset cache:', cache)
+            os.remove(cache)
+
     model.reset_data(force=True)
 
 

bookmarks/common/core.py

@@ -465,19 +465,21 @@ def get_sequence_and_shot(s):
     return seq, shot
 
 
-def get_entry_from_path(path, is_dir=True):
+def get_entry_from_path(path, is_dir=True, force_exists=False):
     """Returns a scandir entry of the given file path.
 
     Args:
         path (str): Path to directory.
         is_dir (bool): Is the path a directory or a file.
+        force_exists (bool): Force skip checking the existence of the path if we know path exist.
 
     Returns:
          scandir.DirEntry: A scandir entry, or None if not found.
 
     """
     file_info = QtCore.QFileInfo(path)
-    if not file_info.exists():
+
+    if not force_exists and not file_info.exists():
         return None
 
     for entry in os.scandir(file_info.dir().path()):
@@ -496,22 +498,21 @@ def get_links(path, section='links/asset'):
     inside job templates.
 
     If a .links file contains two relative paths,
-    `subfolder1/nested_asset1` and `subfolder2/nested_asset2`...
+    `subfolder1/nested_asset1` and `subfolder2/nested_asset2`
 
     .. code-block:: text
 
-        asset/
+        root_asset_folder/
         ├─ .links
         ├─ subfolder1/
         │  ├─ nested_asset1/
         ├─ subfolder2/
         │  ├─ nested_asset2/
 
-    ...two asset will be read, `nested_asset1` and `nested_asset2`
-    (but not the original root `asset`).
+    ...two asset items will be read - `nested_asset1` and `nested_asset2` but not the original root `root_asset_folder`.
 
     Args:
-        path (str): Path to a folder where the link file resides. E.g. an asset root folder.
+        path (str): Path to a folder where the link file resides.
         section (str):
             The settings section to look for links in.
             Optional. Defaults to 'links/asset'.

bookmarks/items/asset_items.py

@@ -48,7 +48,6 @@
 from ..tokens import tokens
 
 
-
 class AssetItemViewContextMenu(contextmenu.BaseContextMenu):
     """The context menu associated with :class:`AssetItemView`."""
 
@@ -179,11 +178,6 @@ def init_data(self):
 
         # Let's get the identifier from the bookmark database
         db = database.get(*p)
-        asset_identifier = db.value(
-            source,
-            'identifier',
-            database.BookmarkTable
-        )
 
         # ...and the display token
         display_token = db.value(
@@ -201,19 +195,12 @@ def init_data(self):
         nth = 17
         c = 0
 
-        for entry in self.item_generator(source):
+        for filepath in self.item_generator(source):
             if self._interrupt_requested:
                 break
 
-            filepath = entry.path.replace('\\', '/')
-
-            if asset_identifier:
-                identifier = f'{filepath}/{asset_identifier}'
-                if not os.path.isfile(identifier):
-                    continue
-
             # Progress bar
-            c += 9
+            c += 1
             if not c % nth:
                 common.signals.showStatusBarMessage.emit(
                     f'Loading assets ({c} found)...'
@@ -238,6 +225,7 @@ def init_data(self):
                 seq, shot = common.get_sequence_and_shot(filepath)
                 _display_name = config.expand_tokens(
                     display_token,
+                    use_database=False,
                     server=p[0],
                     job=p[1],
                     root=p[2],
@@ -250,7 +238,6 @@ def init_data(self):
                 if tokens.invalid_token not in _display_name:
                     display_name = _display_name
 
-
             parent_path_role = p + (filename,)
 
             sort_by_name_role = models.DEFAULT_SORT_BY_NAME_ROLE.copy()
@@ -280,7 +267,7 @@ def init_data(self):
                     common.DataTypeRole: t,
                     common.ItemTabRole: common.AssetTab,
                     #
-                    common.EntryRole: [entry, ],
+                    common.EntryRole: [],
                     common.FlagsRole: flags,
                     common.ParentPathRole: parent_path_role,
                     common.DescriptionRole: '',
@@ -307,6 +294,10 @@ def init_data(self):
                 }
             )
 
+        # Cache the list of assets for later use
+        with open(f'{common.active("root", path=True)}/{common.bookmark_cache_dir}/assets.cache', 'w') as f:
+            f.write('\n'.join([v[common.PathRole] for v in data.values()]))
+
         # Explicitly emit `activeChanged` to notify other dependent models
         self.activeChanged.emit(self.active_index())
 
@@ -326,10 +317,21 @@ def source_path(self):
     def item_generator(self, path):
         """Yields the asset items to be processed by :meth:`init_data`.
 
+        Args:
+            path (string): The path to a directory containing asset folders.
+
         Yields:
             DirEntry: Entry instances of valid asset folders.
 
         """
+        # Read from the cache if it exists
+        cache = f'{common.active("root", path=True)}/{common.bookmark_cache_dir}/assets.cache'
+        if os.path.isfile(cache):
+            with open(cache, 'r') as f:
+                for line in f:
+                    yield line.strip()
+            return
+
         try:
             it = os.scandir(path)
         except OSError as e:
@@ -352,17 +354,13 @@ def item_generator(self, path):
             )
 
             for link in links:
-                v = f'{path}/{entry.name}/{link}'
-                _entry = common.get_entry_from_path(v)
-                if not _entry:
-                    log.error(f'Could not get entry from link {v}')
-                    continue
-                yield _entry
+                yield f'{path}/{entry.name}/{link}'
 
+            # Don't yield the asset if it has links
             if links:
                 continue
 
-            yield entry
+            yield entry.path.replace('\\', '/')
 
     def save_active(self):
         """Saves the active item.

bookmarks/items/models.py

@@ -412,10 +412,13 @@ def _drop_properties_action(self, mime, action, row, column, parent):
                 log.error('Could not remove temp file.')
             return
 
-    def item_generator(self):
+    def item_generator(self, path):
         """A generator method used by :func:`init_data` to yield the items the model
         should load.
 
+        Args:
+            path (string): Path to a directory.
+
         Yields:
             DirEntry: os.scandir DirEntry objects.
 

bookmarks/progress.py

@@ -155,6 +155,16 @@ def paint(self, painter, option, index):
         t = common.FileItem
 
         _data = common.get_data(p, k, t)
+
+        if not _data:
+            return
+        if source_index.row() not in _data:
+            return
+        if not _data[source_index.row()][common.AssetProgressRole]:
+            return
+        if index.column() - 1 not in _data[source_index.row()][common.AssetProgressRole]:
+            return
+
         data = _data[source_index.row()][common.AssetProgressRole][index.column() - 1]
 
         right_edge = self._draw_background(painter, option, data)

bookmarks/threads/workers.py

@@ -570,6 +570,15 @@ def _process_data(self, ref):
         # Asset Progress Data
         if len(pp) == 4 and asset_row_data['progress']:
             ref()[common.AssetProgressRole] = asset_row_data['progress']
+        # Asset entry data
+        if len(pp) == 4:
+            ref()[common.EntryRole].append(
+                common.get_entry_from_path(
+                    ref()[common.PathRole],
+                    is_dir=True,
+                    force_exists=True
+                )
+            )
         # ShotGrid status
         if len(pp) <= 4:
             update_shotgun_configured(pp, bookmark_row_data, asset_row_data, ref)

bookmarks/tokens/tokens.py

@@ -40,9 +40,6 @@
 from .. import database
 from .. import log
 
-#: The database column name
-TOKENS_DB_KEY = 'tokens'
-
 FileFormatConfig = 'FileFormatConfig'
 FileNameConfig = 'FileNameConfig'
 PublishConfig = 'PublishConfig'
@@ -598,7 +595,7 @@ def data(self, force=False):
             db = database.get(self.server, self.job, self.root)
             v = db.value(
                 db.source(),
-                TOKENS_DB_KEY,
+                'tokens',
                 database.BookmarkTable
             )
             if not v or not isinstance(v, dict):
@@ -631,7 +628,7 @@ def set_data(self, data):
         db = database.get(self.server, self.job, self.root)
         db.set_value(
             db.source(),
-            TOKENS_DB_KEY,
+            'tokens',
             data,
             table=database.BookmarkTable
         )
@@ -701,7 +698,7 @@ def get_description(self, token, force=False):
         return ''
 
     def expand_tokens(
-            self, s, user=common.get_username(), version='v001',
+            self, s, use_database=True, user=common.get_username(), version='v001',
             host=socket.gethostname(), task='main',
             ext=None, prefix=None, **_kwargs
     ):
@@ -712,12 +709,7 @@ def expand_tokens(
 
         Args:
             s (str): The string containing tokens to be expanded.
-            user (str, optional): Username.
-            version (str, optional): The version string.
-            host (str, optional): The name of the current machine/host.
-            task (str, optional): Task folder name.
-            ext (str, optional): File format extension.
-            prefix (str, optional): Bookmark item prefix.
+            use_database (bool, optional): Use values stored in the database.
             _kwargs (dict, optional): Optional token/value pairs.
 
         Returns:
@@ -726,6 +718,7 @@ def expand_tokens(
 
         """
         kwargs = self.get_tokens(
+            use_database=use_database,
             user=user,
             version=version,
             ver=version,
@@ -756,11 +749,11 @@ def expand_tokens(
             collections.defaultdict(lambda: invalid_token, **kwargs)
         )
 
-    def get_tokens(self, force=False, **kwargs):
+    def get_tokens(self, force=False, use_database=True, **kwargs):
         """Get all available tokens.
 
         Args:
-            force (bool, optional): Force retrieve tokens from the database.
+            force (bool, optional): Force retrieve defined tokens from the database.
 
         Returns:
             dict: A dictionary of token/value pairs.
@@ -791,34 +784,37 @@ def get_tokens(self, force=False, **kwargs):
             tokens[k] = v
 
         # We can also use bookmark item properties as tokens
-        db = database.get(self.server, self.job, self.root)
-        for _k in database.TABLES[database.BookmarkTable]:
-            if _k == 'id':
-                continue
-            if database.TABLES[database.BookmarkTable][_k]['type'] == dict:
-                continue
-            if _k not in kwargs or not kwargs[_k]:
-                _v = db.value(db.source(), _k, database.BookmarkTable)
-                _v = _v if _v else invalid_token
-                tokens[_k] = _v
-
-        # The asset root token will only be available when the asset is manually
-        # specified
-        if 'asset' in kwargs and kwargs['asset']:
-            source = f'{self.server}/{self.job}/{self.root}/{kwargs["asset"]}'
+        if use_database:
+            db = database.get(self.server, self.job, self.root)
 
-            for _k in database.TABLES[database.AssetTable]:
+            for _k in database.TABLES[database.BookmarkTable]:
                 if _k == 'id':
                     continue
-                if database.TABLES[database.AssetTable][_k]['type'] == dict:
+                if database.TABLES[database.BookmarkTable][_k]['type'] == dict:
                     continue
                 if _k not in kwargs or not kwargs[_k]:
-                    _v = db.value(source, _k, database.AssetTable)
+                    _v = db.value(db.source(), _k, database.BookmarkTable)
                     _v = _v if _v else invalid_token
                     tokens[_k] = _v
 
-                    # Let's also override width, height and fps tokens
-                    tokens[_k.replace('asset_', '')] = _v
+        # The asset root token will only be available when the asset is manually
+        # specified
+        if 'asset' in kwargs and kwargs['asset']:
+            source = f'{self.server}/{self.job}/{self.root}/{kwargs["asset"]}'
+
+            if use_database:
+                for _k in database.TABLES[database.AssetTable]:
+                    if _k == 'id':
+                        continue
+                    if database.TABLES[database.AssetTable][_k]['type'] == dict:
+                        continue
+                    if _k not in kwargs or not kwargs[_k]:
+                        _v = db.value(source, _k, database.AssetTable)
+                        _v = _v if _v else invalid_token
+                        tokens[_k] = _v
+
+                        # Let's also override width, height and fps tokens
+                        tokens[_k.replace('asset_', '')] = _v
 
         # We also want to use the path elements as tokens.
         for k in ('server', 'job', 'root', 'asset', 'task'):