Merge pull request #66 from Ljzd-PRO/devel

Bump to v0.4.0

CHANGELOG.md

@@ -1,5 +1,15 @@
 ## Changes
 
-- Use binary prefixes(1Ki=1024, KiB, MiB, GiB, ...) in download speed units
+- Removed `--update-from` flag
+- Set `LoggerConfiguration.path` defaults to `None` (KToolBox will no longer output logs to files by default)
+- Fixed the issue where downloading files with the same name but different suffix simultaneously would fail.
+- Documents update (Coomer)
 
-**Full Changelog**: https://github.com/Ljzd-PRO/KToolBox/compare/v0.3.5...v0.3.6
+- - -
+
+- 删除了 `--update-from` 标志
+- 将 `LoggerConfiguration.path` 的默认值设置为 `None`（KToolBox 默认不再将日志输出到文件）
+- 修复了同时下载具有相同名称但不同后缀的文件会失败的问题。
+- 文档更新（Coomer）
+
+**Full Changelog**: https://github.com/Ljzd-PRO/KToolBox/compare/v0.3.6...v0.4.0

docs/en/coomer.md

@@ -1,8 +1,10 @@
 # Coomer
 
-KToolBox support downloading from Coomer.su / Coomer.party, but currently need to **configure** before it can use.
+KToolBox support downloading from Coomer.su / Coomer.party
 
-You need to set the configuration by `prod.env` dotenv file or system environment variables:
+Files of Coomer.su can be downloaded from Kemono.su / Kemono.party server, so you can download Coomer posts **without modifying configuration**.
+
+If it did not work, you can modify the configuration by `prod.env` dotenv file or system environment variables and then try again:
 ```dotenv
 # For Coomer API
 KTOOLBOX_API__NETLOC=coomer.su

docs/zh/coomer.md

@@ -1,8 +1,10 @@
 # Coomer
 
-KToolBox 支持从 Coomer.su / Coomer.party 下载，但目前在它能够使用之前需要进行 **配置**。
+KToolBox 支持从 Coomer.su / Coomer.party 下载
 
-你需要通过 dotenv文件 `prod.env` 或系统环境变量来设置配置：
+Coomer.su 的文件可以从 Kemono.su / Kemono.party 服务器下载，所以 **无需编辑配置** 即可下载 Coomer 里的作品
+
+如果无效，你可以通过 dotenv文件 `prod.env` 或系统环境变量来修改配置，然后再次尝试：
 ```dotenv
 # Coomer API
 KTOOLBOX_API__NETLOC=coomer.su

ktoolbox/__init__.py

@@ -1,4 +1,4 @@
 __title__ = "KToolBox"
 # noinspection SpellCheckingInspection
 __description__ = "A useful CLI tool for downloading posts in Kemono.party / .su"
-__version__ = "0.3.6"
+__version__ = "0.4.0"

ktoolbox/action/job.py

@@ -6,12 +6,12 @@
 from loguru import logger
 from pathvalidate import sanitize_filename
 
+from ktoolbox._enum import PostFileTypeEnum, DataStorageNameEnum
 from ktoolbox.action import ActionRet, fetch_all_creator_posts, FetchInterruptError
-from ktoolbox.action.utils import generate_post_path_name, filter_posts_by_time, filter_posts_by_indices
+from ktoolbox.action.utils import generate_post_path_name, filter_posts_by_time
 from ktoolbox.api.model import Post, Attachment
 from ktoolbox.api.posts import get_creator_post
 from ktoolbox.configuration import config, PostStructureConfiguration
-from ktoolbox._enum import PostFileTypeEnum, DataStorageNameEnum
 from ktoolbox.job import Job, CreatorIndices
 
 __all__ = ["create_job_from_post", "create_job_from_creator"]
@@ -89,7 +89,6 @@ async def create_job_from_creator(
         creator_id: str,
         path: Path,
         *,
-        update_from: CreatorIndices = None,
         all_pages: bool = False,
         o: int = None,
         save_creator_indices: bool = True,
@@ -103,8 +102,6 @@ async def create_job_from_creator(
     :param service: The service where the post is located
     :param creator_id: The ID of the creator
     :param path: The path for posts to download
-    :param update_from: ``CreatorIndices`` data for update posts from current creator directory, \
-     ``save_creator_indices`` will be enabled if this provided
     :param all_pages: Fetch all pages of posts, ``o`` will be ignored if enabled
     :param o: Result offset, stepping of 50 is enforced
     :param save_creator_indices: Record ``CreatorIndices`` data for update posts from current creator directory
@@ -138,18 +135,13 @@ async def create_job_from_creator(
 
     # Filter posts and generate ``CreatorIndices``
     if not mix_posts:
-        indices = None
-        if update_from:
-            post_list, indices = filter_posts_by_indices(post_list, update_from)
-            logger.info(f"{len(post_list)} posts will be downloaded")
-        elif save_creator_indices:  # It's unnecessary to create indices again when ``update_from`` was provided
+        if save_creator_indices:  # It's unnecessary to create indices again when ``update_from`` was provided
             indices = CreatorIndices(
                 creator_id=creator_id,
                 service=service,
                 posts={post.id: post for post in post_list},
                 posts_path={post.id: path / sanitize_filename(post.title) for post in post_list}
             )
-        if indices:
             async with aiofiles.open(
                     path / DataStorageNameEnum.CreatorIndicesData.value,
                     "w",
@@ -160,12 +152,7 @@ async def create_job_from_creator(
     job_list: List[Job] = []
     for post in post_list:
         # Get post path
-        default_post_path = path if mix_posts else path / generate_post_path_name(post)
-        if update_from:
-            if not (post_path := update_from.posts_path.get(post.id)):
-                post_path = default_post_path
-        else:
-            post_path = default_post_path
+        post_path = path if mix_posts else path / generate_post_path_name(post)
 
         # Generate jobs
         job_list += await create_job_from_post(

ktoolbox/action/search.py

@@ -1,9 +1,9 @@
 from typing import Iterator, List
 
+from ktoolbox._enum import RetCodeEnum
 from ktoolbox.action import ActionRet
 from ktoolbox.api.model import Creator, Post
 from ktoolbox.api.posts import get_creators, get_creator_post
-from ktoolbox._enum import RetCodeEnum
 from ktoolbox.utils import BaseRet, generate_msg
 
 __all__ = ["search_creator", "search_creator_post"]

ktoolbox/api/base.py

@@ -9,8 +9,8 @@
 from tenacity import RetryCallState, wait_fixed, retry_if_result
 from tenacity.stop import stop_base, stop_never, stop_after_attempt
 
-from ktoolbox.configuration import config
 from ktoolbox._enum import RetCodeEnum
+from ktoolbox.configuration import config
 from ktoolbox.utils import BaseRet, generate_msg
 
 __all__ = ["APITenacityStop", "APIRet", "BaseAPI"]

ktoolbox/cli.py

@@ -7,13 +7,13 @@
 from pathvalidate import sanitize_filename
 
 from ktoolbox import __version__
+from ktoolbox._enum import TextEnum
 from ktoolbox.action import create_job_from_post, create_job_from_creator, generate_post_path_name
 from ktoolbox.action import search_creator as search_creator_action, search_creator_post as search_creator_post_action
 from ktoolbox.api.misc import get_app_version
 from ktoolbox.api.posts import get_post as get_post_api
 from ktoolbox.configuration import config
-from ktoolbox._enum import TextEnum
-from ktoolbox.job import JobRunner, CreatorIndices
+from ktoolbox.job import JobRunner
 from ktoolbox.utils import dump_search, parse_webpage_url, generate_msg
 
 __all__ = ["KToolBoxCli"]
@@ -189,7 +189,6 @@ async def sync_creator(
             url: str,
             path: Union[Path, str] = Path("."),
             *,
-            update_from: Path = None,
             save_creator_indices: bool = True,
             mix_posts: bool = None,
             time_range: Tuple[str, str] = None,
@@ -205,7 +204,6 @@ async def sync_creator(
             creator_id: str,
             path: Union[Path, str] = Path("."),
             *,
-            update_from: Path = None,
             save_creator_indices: bool = True,
             mix_posts: bool = None,
             time_range: Tuple[str, str] = None,
@@ -221,7 +219,6 @@ async def sync_creator(
             creator_id: str = None,
             path: Union[Path, str] = Path("."),
             *,
-            update_from: Union[Path, str] = None,
             save_creator_indices: bool = True,
             mix_posts: bool = None,
             start_time: str = None,
@@ -240,8 +237,6 @@ async def sync_creator(
         :param service: The service where the post is located
         :param creator_id: The ID of the creator
         :param path: Download path, default is current directory
-        :param update_from: ``CreatorIndices`` data path for update posts from current creator directory, \
-            ``save_creator_indices`` will be enabled if this provided
         :param save_creator_indices: Record ``CreatorIndices`` data for update posts from current creator directory
         :param mix_posts: Save all files from different posts at same path, \
             ``update_from``, ``save_creator_indices`` will be ignored if enabled
@@ -264,47 +259,35 @@ async def sync_creator(
                 ])
 
         path = path if isinstance(path, Path) else Path(path)
-        if update_from:
-            update_from = update_from if isinstance(update_from, Path) else Path(update_from)
-
-        # Get ``CreatorIndices`` data
-        if update_from:
-            async with aiofiles.open(update_from, encoding="utf-8") as f:
-                indices_text = await f.read()
-            indices = CreatorIndices.model_validate_json(indices_text)
-            creator_path = update_from.parent
-        else:
-            indices = None
 
-            # Get creator name
-            creator_name = creator_id
-            creator_ret = await search_creator_action(id=creator_id, service=service)
-            if creator_ret:
-                creator = next(creator_ret.data, None)
-                if creator:
-                    creator_name = creator.name
-                    logger.info(
-                        generate_msg(
-                            "Got creator information",
-                            name=creator.name,
-                            id=creator.id
-                        )
-                    )
-            else:
-                logger.warning(
+        # Get creator name
+        creator_name = creator_id
+        creator_ret = await search_creator_action(id=creator_id, service=service)
+        if creator_ret:
+            creator = next(creator_ret.data, None)
+            if creator:
+                creator_name = creator.name
+                logger.info(
                     generate_msg(
-                        f"Failed to fetch the name of creator <{creator_id}>, use creator ID as directory name",
-                        detail=creator_ret.message
+                        "Got creator information",
+                        name=creator.name,
+                        id=creator.id
                     )
                 )
-            creator_path = path / sanitize_filename(creator_name)
+        else:
+            logger.warning(
+                generate_msg(
+                    f"Failed to fetch the name of creator <{creator_id}>, use creator ID as directory name",
+                    detail=creator_ret.message
+                )
+            )
+        creator_path = path / sanitize_filename(creator_name)
 
         creator_path.mkdir(exist_ok=True)
         ret = await create_job_from_creator(
             service=service,
             creator_id=creator_id,
             path=creator_path,
-            update_from=indices,
             all_pages=True,
             save_creator_indices=save_creator_indices,
             mix_posts=mix_posts,

ktoolbox/configuration.py

@@ -119,7 +119,7 @@ class LoggerConfiguration(BaseModel):
     :ivar level: Log filter level
     :ivar rotation: Log rotation
     """
-    path: Optional[Path] = Path("logs")
+    path: Optional[Path] = None
     level: Union[str, int] = logging.DEBUG
     rotation: Union[str, int, datetime.time, datetime.timedelta] = "1 week"
 

ktoolbox/downloader/downloader.py

@@ -14,9 +14,9 @@
 from tenacity.stop import stop_after_attempt, stop_never
 from tqdm import tqdm as std_tqdm
 
+from ktoolbox._enum import RetCodeEnum
 from ktoolbox.configuration import config
 from ktoolbox.downloader import DownloaderRet
-from ktoolbox._enum import RetCodeEnum
 from ktoolbox.utils import filename_from_headers, generate_msg
 
 __all__ = ["Downloader"]
@@ -171,7 +171,8 @@ async def run(
                     self._filename = filename
 
                     # Download
-                    temp_filepath = (self._path / filename).with_suffix(f".{config.downloader.temp_suffix}")
+                    temp_filepath = (self._path / filename).with_suffix(
+                        f"{Path(filename).suffix}.{config.downloader.temp_suffix}")
                     total_size = int(length_str) if (length_str := res.headers.get("Content-Length")) else None
                     async with aiofiles.open(str(temp_filepath), "wb", self._buffer_size) as f:
                         chunk_iterator = res.aiter_bytes(self._chunk_size)

ktoolbox/job/model.py

@@ -3,8 +3,8 @@
 
 from pydantic import BaseModel
 
-from ktoolbox.api.model import Post
 from ktoolbox._enum import PostFileTypeEnum
+from ktoolbox.api.model import Post
 from ktoolbox.model import BaseKToolBoxData
 
 __all__ = ["Job", "JobListData", "CreatorIndices"]

ktoolbox/job/runner.py

@@ -8,9 +8,9 @@
 from loguru import logger
 from tqdm import tqdm as std_tqdm
 
+from ktoolbox._enum import RetCodeEnum
 from ktoolbox.configuration import config
 from ktoolbox.downloader import Downloader
-from ktoolbox._enum import RetCodeEnum
 from ktoolbox.job import Job
 from ktoolbox.utils import generate_msg
 
@@ -70,7 +70,11 @@ def processing_size(self) -> int:
         return len(self._downloaders_with_task) - self.done_size
 
     async def processor(self) -> int:
-        """Process each job in ``self._job_queue``"""
+        """
+        Process each job in ``self._job_queue``
+
+        :return: Number of jobs that failed
+        """
         failed_num = 0
         while not self._job_queue.empty():
             job = await self._job_queue.get()
@@ -159,6 +163,7 @@ async def start(self):
             logger.warning(generate_msg(f"{failed_num} jobs failed, download finished"))
         else:
             logger.success(generate_msg("All jobs in queue finished"))
+        return failed_num
 
     async def add_jobs(self, *jobs: Job):
         """Add jobs to ``self._job_queue``"""

ktoolbox/utils.py

@@ -10,8 +10,8 @@
 from loguru import logger
 from pydantic import BaseModel, ConfigDict
 
-from ktoolbox.configuration import config
 from ktoolbox._enum import RetCodeEnum, DataStorageNameEnum
+from ktoolbox.configuration import config
 from ktoolbox.model import SearchResult
 
 __all__ = [