Add pyproject.toml, refactor pickle funcs, and release v0.6.1

.github/workflows/linting.yml

@@ -23,7 +23,7 @@ jobs:
 
             - name: Install Flake8
               run: |
-                pip install flake8
+                pip install flake8 flake8-pyproject
 
             - name: Run linting
               run: |

.github/workflows/publish_to_PyPI.yml

@@ -32,14 +32,11 @@ jobs:
       with:
         python-version: '3.10'
         check-latest: true
-        cache-dependency-path: |
-          requirements.txt
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install build wheel
-        pip install -r requirements.txt
+        pip install build
 
     - name: Fetch the test environment details
       run: |
@@ -48,7 +45,7 @@ jobs:
 
     - name: Build package
       run: |
-        python -m build --no-isolation
+        python -m build
 
     - name: Publish the new package to PyPI
       uses: pypa/gh-action-pypi-publish@v1.8.7

.pre-commit-config.yaml

@@ -18,6 +18,6 @@ repos:
         hooks:
             -   id: flake8
                 args: [
-                    --max-line-length=120, # refer to setup.cfg
+                    --max-line-length=120, # refer to pyproject.toml
                     --extend-ignore=E203, # why ignore E203? Refer to https://github.com/PyCQA/pycodestyle/issues/373
                 ]

pyproject.toml

@@ -0,0 +1,75 @@
+[build-system]
+requires = ["setuptools>=71"]
+
+[project]
+name = "tsdb"
+description = "TSDB (Time Series Data Beans): a Python toolbox helping load 172 open-source time-series datasets"
+authors = [{ name = "Wenjie Du", email = "wenjay.du@gmail.com" }]
+dynamic = ["version", "readme", "dependencies"]
+license = { file = "LICENSE" }
+requires-python = ">=3.8"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Education",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: BSD License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries :: Application Frameworks",
+]
+keywords = [
+    "data mining",
+    "time series",
+    "time-series analysis",
+    "time-series database",
+    "time-series datasets",
+    "database",
+    "datasets",
+    "dataset downloading",
+    "imputation",
+    "classification",
+    "forecasting",
+    "partially observed",
+    "irregularly sampled",
+    "partially-observed time series",
+    "incomplete time series",
+    "missing data",
+    "missing values",
+    "pypots",
+]
+
+[project.urls]
+Source = "https://github.com/WenjieDu/TSDB"
+Homepage = "https://pypots.com"
+Documentation = "https://docs.pypots.com"
+"Bug Tracker" = "https://github.com/WenjieDu/TSDB/issues"
+Download = "https://github.com/WenjieDu/TSDB/archive/main.zip"
+
+[tool.setuptools.packages.find]
+exclude = [
+    "docs*",
+    "test*",
+    "dataset_profiles*",
+]
+
+[tool.setuptools.dynamic]
+version = { attr = "tsdb.version.__version__" }
+readme = { file = "README.md", content-type = "text/markdown" }
+dependencies = { file = "requirements.txt" }
+
+[tool.flake8]
+# People may argue that coding style is personal. This may be true if the project is personal and one works like a
+# hermit, but to PyPOTS and its community, the answer is NO.
+# We use Black and Flake8 to lint code style and keep the style consistent across all commits and pull requests.
+# Black only reformats the code, and Flake8 is necessary for checking for some other issues not covered by Black.
+
+# The Black line length is default as 88, while the default of Flake8 is 79. However, considering our monitors are
+# much more advanced nowadays, I extend the maximum line length to 120, like other project e.g. transformers. People
+# who prefer the default setting can keep using 88 or 79 while coding. Please ensure your code lines not exceeding 120.
+max-line-length = 120
+# why ignore E203? Refer to https://github.com/PyCQA/pycodestyle/issues/373
+extend-ignore = """
+    E203
+"""

tsdb/__init__.py

@@ -5,24 +5,6 @@
 # Created by Wenjie Du <wenjay.du@gmail.com>
 # License: BSD-3-Clause
 
-# TSDB version
-#
-# PEP0440 compatible formatted version, see:
-# https://www.python.org/dev/peps/pep-0440/
-# Generic release markers:
-# X.Y
-# X.Y.Z # For bugfix releases
-#
-# Admissible pre-release markers:
-# X.YaN # Alpha release
-# X.YbN # Beta release
-# X.YrcN # Release Candidate
-# X.Y # Final release
-#
-# Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
-# 'X.Y.dev0' is the canonical version of 'X.Y.dev'
-__version__ = "0.6"
-
 from .data_processing import (
     CACHED_DATASET_DIR,
     list,
@@ -38,6 +20,7 @@
     migrate,
     migrate_cache,
 )
+from .version import __version__
 
 __all__ = [
     "__version__",

tsdb/utils/file.py

@@ -9,7 +9,6 @@
 import os
 import pickle
 import shutil
-from typing import Optional
 
 from .config import read_configs, write_configs
 from .logging import logger
@@ -51,7 +50,44 @@ def check_path(
     return checked_path
 
 
-def pickle_dump(data: object, path: str) -> Optional[str]:
+def extract_parent_dir(path: str) -> str:
+    """Extract the given path's parent directory.
+
+    Parameters
+    ----------
+    path :
+        The path for extracting.
+
+    Returns
+    -------
+    parent_dir :
+        The path to the parent dir of the given path.
+
+    """
+    parent_dir = os.path.abspath(os.path.join(path, ".."))
+    return parent_dir
+
+
+def create_dir_if_not_exist(path: str, is_dir: bool = True) -> None:
+    """Create the given directory if it doesn't exist.
+
+    Parameters
+    ----------
+    path :
+        The path for check.
+
+    is_dir :
+        Whether the given path is to a directory. If `is_dir` is False, the given path is to a file or an object,
+        then this file's parent directory will be checked.
+
+    """
+    path = extract_parent_dir(path) if not is_dir else path
+    if not os.path.exists(path):
+        os.makedirs(path, exist_ok=True)
+        logger.info(f"Successfully created the given path {path}")
+
+
+def pickle_dump(data: object, path: str) -> None:
     """Pickle the given object.
 
     Parameters
@@ -67,17 +103,18 @@ def pickle_dump(data: object, path: str) -> Optional[str]:
     `path` if succeed else None
 
     """
-    # check the given path
-    path = check_path(path)
-
     try:
+        # help create the parent dir if not exist
+        create_dir_if_not_exist(extract_parent_dir(path))
         with open(path, "wb") as f:
             pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
-    except pickle.PicklingError:
-        logger.info("Pickling failed. No cache will be saved.")
-        return None
-    logger.info(f"Successfully saved to {path}")
-    return path
+        logger.info(f"Successfully saved to {path}")
+    except Exception as e:
+        logger.error(
+            f"❌ Pickling failed. No cache data saved. Investigate the error below:\n{e}"
+        )
+
+    return None
 
 
 def pickle_load(path: str) -> object:
@@ -94,13 +131,15 @@ def pickle_load(path: str) -> object:
         Pickled object.
 
     """
-    # check the given path
-    path = check_path(path, check_exists=True)
     try:
         with open(path, "rb") as f:
             data = pickle.load(f)
-    except pickle.UnpicklingError as e:
-        logger.info("Cached data corrupted. Aborting...\n" f"{e}")
+    except Exception as e:
+        logger.error(
+            f"❌ Loading data failed. Operation aborted. Investigate the error below:\n{e}"
+        )
+        return None
+
     return data
 
 

tsdb/version.py

@@ -0,0 +1,24 @@
+"""
+
+"""
+
+# Created by Wenjie Du <wenjay.du@gmail.com>
+# License: BSD-3-Clause
+
+# TSDB version
+#
+# PEP0440 compatible formatted version, see:
+# https://www.python.org/dev/peps/pep-0440/
+# Generic release markers:
+# X.Y
+# X.Y.Z # For bugfix releases
+#
+# Admissible pre-release markers:
+# X.YaN # Alpha release
+# X.YbN # Beta release
+# X.YrcN # Release Candidate
+# X.Y # Final release
+#
+# Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
+# 'X.Y.dev0' is the canonical version of 'X.Y.dev'
+__version__ = "0.6.1"