From 21d982646e099780083ec8c5bc45e6f439539cb3 Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Sun, 28 Jan 2024 09:30:03 +0100 Subject: [PATCH] Fix code related to import statements with isort The fix was done in a virtual Python environment: pip install isort isort . It orders the import statements and fixes their formatting. Signed-off-by: Stefan Weil --- dev/analyze_profile.py | 1 + dev/groupid_to_structmap.py | 3 +- src/ocrd/__init__.py | 10 +- src/ocrd/cli/__init__.py | 10 +- src/ocrd/cli/bashlib.py | 25 ++-- src/ocrd/cli/log.py | 7 +- src/ocrd/cli/network.py | 12 +- src/ocrd/cli/ocrd_tool.py | 18 +-- src/ocrd/cli/process.py | 3 +- src/ocrd/cli/resmgr.py | 17 +-- src/ocrd/cli/validate.py | 17 +-- src/ocrd/cli/workspace.py | 24 ++-- src/ocrd/cli/zip.py | 3 +- src/ocrd/decorators/__init__.py | 26 ++-- src/ocrd/decorators/loglevel_option.py | 1 + src/ocrd/decorators/mets_find_options.py | 1 + src/ocrd/decorators/ocrd_cli_options.py | 15 +-- src/ocrd/decorators/parameter_option.py | 1 + src/ocrd/mets_server.py | 21 +-- src/ocrd/processor/__init__.py | 10 +- src/ocrd/processor/base.py | 37 +++--- src/ocrd/processor/builtin/dummy_processor.py | 17 +-- src/ocrd/processor/helpers.py | 13 +- src/ocrd/resolver.py | 16 +-- src/ocrd/resource_manager.py | 28 ++-- src/ocrd/task_sequence.py | 12 +- src/ocrd/workspace.py | 53 +++----- src/ocrd/workspace_backup.py | 9 +- src/ocrd/workspace_bagger.py | 33 +++-- src/ocrd_modelfactory/__init__.py | 14 +- src/ocrd_models/__init__.py | 4 +- src/ocrd_models/constants.py | 3 +- src/ocrd_models/ocrd_agent.py | 4 +- src/ocrd_models/ocrd_exif.py | 8 +- src/ocrd_models/ocrd_file.py | 6 +- src/ocrd_models/ocrd_mets.py | 39 ++---- src/ocrd_models/ocrd_page.py | 87 ++++-------- src/ocrd_models/ocrd_page_generateds.py | 22 +-- src/ocrd_models/ocrd_xml_base.py | 2 +- src/ocrd_models/utils.py | 1 + src/ocrd_network/__init__.py | 7 +- src/ocrd_network/cli/client.py | 9 +- src/ocrd_network/cli/processing_server.py | 6 +- src/ocrd_network/cli/processing_worker.py | 7 +- src/ocrd_network/cli/processor_server.py | 7 +- src/ocrd_network/client.py | 1 + src/ocrd_network/database.py | 15 +-- src/ocrd_network/deployer.py | 32 ++--- src/ocrd_network/deployment_utils.py | 8 +- src/ocrd_network/logging.py | 3 +- src/ocrd_network/models/__init__.py | 12 +- src/ocrd_network/models/job.py | 5 +- src/ocrd_network/models/messages.py | 4 +- src/ocrd_network/models/ocrd_tool.py | 3 +- src/ocrd_network/models/workspace.py | 3 +- src/ocrd_network/param_validators.py | 5 +- src/ocrd_network/process_helpers.py | 4 +- src/ocrd_network/processing_server.py | 72 +++------- src/ocrd_network/processing_worker.py | 35 ++--- src/ocrd_network/processor_server.py | 50 ++----- src/ocrd_network/rabbitmq_utils/__init__.py | 7 +- src/ocrd_network/rabbitmq_utils/connector.py | 20 ++- src/ocrd_network/rabbitmq_utils/consumer.py | 13 +- .../rabbitmq_utils/ocrd_messages.py | 2 + src/ocrd_network/rabbitmq_utils/publisher.py | 14 +- src/ocrd_network/runtime_data.py | 8 +- src/ocrd_network/server_cache.py | 12 +- src/ocrd_network/server_utils.py | 17 +-- src/ocrd_network/utils.py | 10 +- src/ocrd_page_user_methods.py | 3 +- .../get_AllAlternativeImagePaths.py | 4 +- src/ocrd_page_user_methods/get_AllRegions.py | 3 +- src/ocrd_utils/__init__.py | 125 ++++-------------- src/ocrd_utils/constants.py | 3 +- src/ocrd_utils/deprecate.py | 1 + src/ocrd_utils/image.py | 4 +- src/ocrd_utils/introspect.py | 1 + src/ocrd_utils/logging.py | 10 +- src/ocrd_utils/os.py | 21 +-- src/ocrd_utils/str.py | 8 +- src/ocrd_validators/__init__.py | 14 +- src/ocrd_validators/constants.py | 2 +- src/ocrd_validators/json_validator.py | 4 +- .../ocrd_network_message_validator.py | 5 +- src/ocrd_validators/ocrd_zip_validator.py | 14 +- src/ocrd_validators/page_validator.py | 39 ++---- src/ocrd_validators/parameter_validator.py | 2 +- .../resource_list_validator.py | 2 +- src/ocrd_validators/workspace_validator.py | 9 +- src/ocrd_validators/xsd_mets_validator.py | 3 +- src/ocrd_validators/xsd_page_validator.py | 3 +- src/ocrd_validators/xsd_validator.py | 1 + tests/base.py | 21 +-- tests/cli/test_bashlib.py | 25 ++-- tests/cli/test_log.py | 14 +- tests/cli/test_process.py | 5 +- tests/cli/test_resmgr.py | 5 +- tests/cli/test_validate.py | 12 +- tests/cli/test_workspace.py | 22 +-- tests/data/__init__.py | 1 + tests/data/wf_testcase.py | 6 +- tests/model/mets_bench_extreme.py | 4 +- tests/model/mets_bench_extreme_additional.py | 4 +- tests/model/test_agent.py | 8 +- tests/model/test_exif.py | 10 +- tests/model/test_ocrd_file.py | 11 +- tests/model/test_ocrd_mets.py | 33 ++--- tests/model/test_ocrd_mets_bench.py | 8 +- tests/model/test_ocrd_page.py | 26 +--- tests/processor/test_ocrd_dummy.py | 10 +- tests/processor/test_processor.py | 19 ++- tests/test_decorators.py | 24 ++-- tests/test_logging.py | 23 ++-- tests/test_logging_conf.py | 9 +- tests/test_mets_server.py | 19 +-- tests/test_model_factory.py | 13 +- tests/test_resolver.py | 26 +--- tests/test_resolver_oai.py | 12 +- tests/test_resource_manager.py | 9 +- tests/test_task_sequence.py | 18 +-- tests/test_utils.py | 58 +++----- tests/test_version.py | 3 +- tests/test_workspace.py | 38 +++--- tests/test_workspace_remove.py | 12 +- tests/utils/test_config.py | 5 +- tests/utils/test_deprecate.py | 3 +- tests/utils/test_image.py | 4 +- tests/utils/test_os.py | 21 ++- tests/validator/test_json_validator.py | 4 +- tests/validator/test_ocrd_tool_validator.py | 4 +- tests/validator/test_ocrd_zip_validator.py | 11 +- tests/validator/test_page_validator.py | 9 +- tests/validator/test_parameter_validator.py | 3 +- tests/validator/test_report.py | 4 +- .../validator/test_resource_list_validator.py | 4 +- tests/validator/test_workspace_backup.py | 7 +- tests/validator/test_workspace_bagger.py | 20 +-- tests/validator/test_workspace_validator.py | 10 +- tests/validator/test_xsd_validator.py | 5 +- 139 files changed, 813 insertions(+), 1114 deletions(-) diff --git a/dev/analyze_profile.py b/dev/analyze_profile.py index 8c142b07ba..6bfae2efa6 100644 --- a/dev/analyze_profile.py +++ b/dev/analyze_profile.py @@ -1,4 +1,5 @@ import pstats + p = pstats.Stats('profile') p.strip_dirs() p.sort_stats('tottime') diff --git a/dev/groupid_to_structmap.py b/dev/groupid_to_structmap.py index d06e8a9e94..19818144af 100755 --- a/dev/groupid_to_structmap.py +++ b/dev/groupid_to_structmap.py @@ -1,7 +1,8 @@ #!/usr/bin/env python -from sys import argv from os.path import isfile +from sys import argv + from ocrd.model import OcrdMets fname = argv[1] diff --git a/src/ocrd/__init__.py b/src/ocrd/__init__.py index 62b6ffbc0a..8d02f8a55b 100644 --- a/src/ocrd/__init__.py +++ b/src/ocrd/__init__.py @@ -14,11 +14,11 @@ """ -from ocrd.processor.base import run_processor, run_cli, Processor -from ocrd_models import OcrdMets, OcrdExif, OcrdFile, OcrdAgent +from ocrd.mets_server import OcrdMetsServer +from ocrd.processor.base import Processor, run_cli, run_processor from ocrd.resolver import Resolver -from ocrd_validators import * +from ocrd.resource_manager import OcrdResourceManager from ocrd.workspace import Workspace from ocrd.workspace_backup import WorkspaceBackupManager -from ocrd.resource_manager import OcrdResourceManager -from ocrd.mets_server import OcrdMetsServer +from ocrd_models import OcrdAgent, OcrdExif, OcrdFile, OcrdMets +from ocrd_validators import * diff --git a/src/ocrd/cli/__init__.py b/src/ocrd/cli/__init__.py index 97cff1db7f..f722b07ed3 100644 --- a/src/ocrd/cli/__init__.py +++ b/src/ocrd/cli/__init__.py @@ -6,6 +6,7 @@ :nested: short """ import re + import click from ocrd_utils import config @@ -68,16 +69,17 @@ def get_help(self, ctx): return CommandWithReplacedHelp +from ocrd.cli.bashlib import bashlib_cli from ocrd.cli.ocrd_tool import ocrd_tool_cli -from ocrd.cli.workspace import workspace_cli from ocrd.cli.process import process_cli -from ocrd.cli.bashlib import bashlib_cli -from ocrd.cli.validate import validate_cli from ocrd.cli.resmgr import resmgr_cli +from ocrd.cli.validate import validate_cli +from ocrd.cli.workspace import workspace_cli from ocrd.decorators import ocrd_loglevel -from .zip import zip_cli + from .log import log_cli from .network import network_cli +from .zip import zip_cli @click.group(epilog=_epilog) diff --git a/src/ocrd/cli/bashlib.py b/src/ocrd/cli/bashlib.py index 5746151c72..83c4831413 100644 --- a/src/ocrd/cli/bashlib.py +++ b/src/ocrd/cli/bashlib.py @@ -7,29 +7,24 @@ """ from __future__ import print_function + import sys from os.path import isfile + import click -from ocrd.constants import BASHLIB_FILENAME import ocrd.constants -import ocrd_utils.constants -from ocrd_utils.constants import DEFAULT_METS_BASENAME import ocrd_models.constants +import ocrd_utils.constants import ocrd_validators.constants -from ocrd.decorators import ( - parameter_option, - parameter_override_option, - ocrd_loglevel -) -from ocrd_utils import ( - is_local_filename, - get_local_filename, - initLogging, - make_file_id -) -from ocrd.resolver import Resolver +from ocrd.constants import BASHLIB_FILENAME +from ocrd.decorators import (ocrd_loglevel, parameter_option, + parameter_override_option) from ocrd.processor import Processor +from ocrd.resolver import Resolver +from ocrd_utils import (get_local_filename, initLogging, is_local_filename, + make_file_id) +from ocrd_utils.constants import DEFAULT_METS_BASENAME # ---------------------------------------------------------------------- # ocrd bashlib diff --git a/src/ocrd/cli/log.py b/src/ocrd/cli/log.py index 560254f794..38d953edf7 100644 --- a/src/ocrd/cli/log.py +++ b/src/ocrd/cli/log.py @@ -5,10 +5,13 @@ :prog: ocrd log :nested: full """ -import click -from ocrd_utils import initLogging, getLogger, getLevelName import logging +import click + +from ocrd_utils import getLevelName, getLogger, initLogging + + class LogCtx(): def __init__(self, name): diff --git a/src/ocrd/cli/network.py b/src/ocrd/cli/network.py index 9203b8840e..42280c8247 100644 --- a/src/ocrd/cli/network.py +++ b/src/ocrd/cli/network.py @@ -6,15 +6,13 @@ :nested: full """ -import click import logging + +import click + +from ocrd_network.cli import (client_cli, processing_server_cli, + processing_worker_cli, processor_server_cli) from ocrd_utils import initLogging -from ocrd_network.cli import ( - client_cli, - processing_server_cli, - processing_worker_cli, - processor_server_cli, -) @click.group("network") diff --git a/src/ocrd/cli/ocrd_tool.py b/src/ocrd/cli/ocrd_tool.py index 4d0983751c..279338a1e8 100644 --- a/src/ocrd/cli/ocrd_tool.py +++ b/src/ocrd/cli/ocrd_tool.py @@ -6,21 +6,21 @@ :nested: full """ -from inspect import getmodule -from json import dumps import codecs -import sys import os +import sys +from inspect import getmodule +from json import dumps + import click from ocrd.decorators import parameter_option, parameter_override_option from ocrd.processor import Processor -from ocrd_utils import ( - set_json_key_value_overrides, - VERSION as OCRD_VERSION, - parse_json_string_with_comments as loads -) -from ocrd_validators import ParameterValidator, OcrdToolValidator +from ocrd_utils import VERSION as OCRD_VERSION +from ocrd_utils import parse_json_string_with_comments as loads +from ocrd_utils import set_json_key_value_overrides +from ocrd_validators import OcrdToolValidator, ParameterValidator + class OcrdToolCtx(): diff --git a/src/ocrd/cli/process.py b/src/ocrd/cli/process.py index b71b74d096..89d2d62727 100644 --- a/src/ocrd/cli/process.py +++ b/src/ocrd/cli/process.py @@ -8,11 +8,12 @@ """ import click -from ocrd_utils import getLogger, initLogging, DEFAULT_METS_BASENAME from ocrd.task_sequence import run_tasks +from ocrd_utils import DEFAULT_METS_BASENAME, getLogger, initLogging from ..decorators import ocrd_loglevel + # ---------------------------------------------------------------------- # ocrd process # ---------------------------------------------------------------------- diff --git a/src/ocrd/cli/resmgr.py b/src/ocrd/cli/resmgr.py index 2e744a8958..d9a4e719dd 100644 --- a/src/ocrd/cli/resmgr.py +++ b/src/ocrd/cli/resmgr.py @@ -6,25 +6,20 @@ :nested: full """ import sys -from pathlib import Path from distutils.spawn import find_executable as which -from yaml import safe_load, safe_dump +from pathlib import Path -import requests import click +import requests +from yaml import safe_dump, safe_load -from ocrd_utils import ( - initLogging, - directory_size, - getLogger, - get_ocrd_tool_json, - get_moduledir, - RESOURCE_LOCATIONS, -) from ocrd.constants import RESOURCE_USER_LIST_COMMENT +from ocrd_utils import (RESOURCE_LOCATIONS, directory_size, get_moduledir, + get_ocrd_tool_json, getLogger, initLogging) from ..resource_manager import OcrdResourceManager + def print_resources(executable, reslist, resmgr): print('%s' % executable) for resdict in reslist: diff --git a/src/ocrd/cli/validate.py b/src/ocrd/cli/validate.py index b26803d053..b3c0a79b5d 100644 --- a/src/ocrd/cli/validate.py +++ b/src/ocrd/cli/validate.py @@ -5,23 +5,20 @@ :prog: ocrd validate :nested: full """ +import codecs import sys +from json import loads import click -from json import loads -import codecs from ocrd import Resolver, Workspace from ocrd.task_sequence import ProcessorTask, validate_tasks +from ocrd_utils import (DEFAULT_METS_BASENAME, initLogging, + parse_json_string_or_file) +from ocrd_validators import (OcrdToolValidator, OcrdZipValidator, + PageValidator, ParameterValidator, + WorkspaceValidator) -from ocrd_utils import initLogging, parse_json_string_or_file, DEFAULT_METS_BASENAME -from ocrd_validators import ( - OcrdToolValidator, - OcrdZipValidator, - PageValidator, - ParameterValidator, - WorkspaceValidator, -) def _inform_of_result(report): if not report.is_valid: diff --git a/src/ocrd/cli/workspace.py b/src/ocrd/cli/workspace.py index 318c1e971d..e89160313b 100644 --- a/src/ocrd/cli/workspace.py +++ b/src/ocrd/cli/workspace.py @@ -6,22 +6,26 @@ :nested: full """ import os -from os import getcwd -from os.path import relpath, exists, join, isabs -from pathlib import Path -from json import loads, dumps -import sys -from glob import glob # XXX pathlib.Path.glob does not support absolute globs import re +import sys import time -import numpy as np +from glob import glob # XXX pathlib.Path.glob does not support absolute globs +from json import dumps, loads +from os import getcwd +from os.path import exists, isabs, join, relpath +from pathlib import Path import click +import numpy as np -from ocrd import Resolver, Workspace, WorkspaceValidator, WorkspaceBackupManager -from ocrd.mets_server import OcrdMetsServer -from ocrd_utils import getLogger, initLogging, pushd_popd, EXT_TO_MIME, safe_filename, parse_json_string_or_file, partition_list, DEFAULT_METS_BASENAME +from ocrd import (Resolver, Workspace, WorkspaceBackupManager, + WorkspaceValidator) from ocrd.decorators import mets_find_options +from ocrd.mets_server import OcrdMetsServer +from ocrd_utils import (DEFAULT_METS_BASENAME, EXT_TO_MIME, getLogger, + initLogging, parse_json_string_or_file, partition_list, + pushd_popd, safe_filename) + from . import command_with_replaced_help diff --git a/src/ocrd/cli/zip.py b/src/ocrd/cli/zip.py index 7db18b09c3..51c7c49ca0 100644 --- a/src/ocrd/cli/zip.py +++ b/src/ocrd/cli/zip.py @@ -9,13 +9,14 @@ import click -from ocrd_utils import initLogging, DEFAULT_METS_BASENAME +from ocrd_utils import DEFAULT_METS_BASENAME, initLogging from ocrd_validators import OcrdZipValidator from ..resolver import Resolver from ..workspace import Workspace from ..workspace_bagger import WorkspaceBagger + @click.group("zip") def zip_cli(): """ diff --git a/src/ocrd/decorators/__init__.py b/src/ocrd/decorators/__init__.py index ecfef5dbb0..e4483e4c3a 100644 --- a/src/ocrd/decorators/__init__.py +++ b/src/ocrd/decorators/__init__.py @@ -1,24 +1,18 @@ import sys -from ocrd_utils import ( - config, - initLogging, - is_local_filename, - get_local_filename, - getLogger, - parse_json_string_with_comments, - set_json_key_value_overrides, -) +from ocrd_network import (NETWORK_AGENT_SERVER, NETWORK_AGENT_WORKER, + ProcessingWorker, ProcessorServer) +from ocrd_utils import (config, get_local_filename, getLogger, initLogging, + is_local_filename, parse_json_string_with_comments, + set_json_key_value_overrides) from ocrd_validators import WorkspaceValidator -from ocrd_network import ProcessingWorker, ProcessorServer, NETWORK_AGENT_SERVER, NETWORK_AGENT_WORKER -from ..resolver import Resolver from ..processor.base import run_processor - +from ..resolver import Resolver from .loglevel_option import ocrd_loglevel -from .parameter_option import parameter_option, parameter_override_option -from .ocrd_cli_options import ocrd_cli_options from .mets_find_options import mets_find_options +from .ocrd_cli_options import ocrd_cli_options +from .parameter_option import parameter_option, parameter_override_option SUBCOMMANDS = [NETWORK_AGENT_WORKER, NETWORK_AGENT_SERVER] @@ -113,10 +107,10 @@ def ocrd_cli_wrap_processor( if not profile_file and config.is_set('OCRD_PROFILE_FILE'): profile_file = config.OCRD_PROFILE_FILE if profile or profile_file: + import atexit import cProfile - import pstats import io - import atexit + import pstats print("Profiling...") pr = cProfile.Profile() pr.enable() diff --git a/src/ocrd/decorators/loglevel_option.py b/src/ocrd/decorators/loglevel_option.py index 9edd013454..54ca702090 100644 --- a/src/ocrd/decorators/loglevel_option.py +++ b/src/ocrd/decorators/loglevel_option.py @@ -1,4 +1,5 @@ import click + from ocrd_utils.logging import setOverrideLogLevel __all__ = ['ocrd_loglevel'] diff --git a/src/ocrd/decorators/mets_find_options.py b/src/ocrd/decorators/mets_find_options.py index f604605d3f..74268cfedb 100644 --- a/src/ocrd/decorators/mets_find_options.py +++ b/src/ocrd/decorators/mets_find_options.py @@ -1,5 +1,6 @@ from click import option + def mets_find_options(f): for opt in [ option('-G', '--file-grp', help="fileGrp USE", metavar='FILTER'), diff --git a/src/ocrd/decorators/ocrd_cli_options.py b/src/ocrd/decorators/ocrd_cli_options.py index d1d3a96243..1e761ad9a4 100644 --- a/src/ocrd/decorators/ocrd_cli_options.py +++ b/src/ocrd/decorators/ocrd_cli_options.py @@ -1,14 +1,13 @@ import click -from click import option, Path, group, command, argument +from click import Path, argument, command, group, option + +from ocrd_network import (NETWORK_AGENT_SERVER, NETWORK_AGENT_WORKER, + DatabaseParamType, QueueServerParamType, + ServerAddressParamType) from ocrd_utils import DEFAULT_METS_BASENAME -from ocrd_network import NETWORK_AGENT_SERVER, NETWORK_AGENT_WORKER -from .parameter_option import parameter_option, parameter_override_option + from .loglevel_option import loglevel_option -from ocrd_network import ( - DatabaseParamType, - ServerAddressParamType, - QueueServerParamType -) +from .parameter_option import parameter_option, parameter_override_option def ocrd_cli_options(f): diff --git a/src/ocrd/decorators/parameter_option.py b/src/ocrd/decorators/parameter_option.py index 7fb989be23..a814e4ad52 100644 --- a/src/ocrd/decorators/parameter_option.py +++ b/src/ocrd/decorators/parameter_option.py @@ -1,4 +1,5 @@ from click import option + from ocrd_utils import parse_json_string_or_file __all__ = ['parameter_option', 'parameter_override_option'] diff --git a/src/ocrd/mets_server.py b/src/ocrd/mets_server.py index 08011557b5..7ad0437da5 100644 --- a/src/ocrd/mets_server.py +++ b/src/ocrd/mets_server.py @@ -2,24 +2,25 @@ # METS server functionality """ import re -from os import environ, _exit, chmod +import socket from io import BytesIO -from typing import Any, Dict, Optional, Union, List, Tuple +from os import _exit, chmod, environ from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union from urllib.parse import urlparse -import socket -from fastapi import FastAPI, Request, File, Form, Response +import uvicorn +from fastapi import FastAPI, File, Form, Request, Response from fastapi.responses import JSONResponse -from requests import request, Session as requests_session +from pydantic import BaseModel, Field, ValidationError +from requests import Session as requests_session +from requests import request from requests.exceptions import ConnectionError from requests_unixsocket import Session as requests_unixsocket_session -from pydantic import BaseModel, Field, ValidationError - -import uvicorn -from ocrd_models import OcrdMets, OcrdFile, ClientSideOcrdFile, OcrdAgent, ClientSideOcrdAgent -from ocrd_utils import initLogging, getLogger, deprecated_alias +from ocrd_models import (ClientSideOcrdAgent, ClientSideOcrdFile, OcrdAgent, + OcrdFile, OcrdMets) +from ocrd_utils import deprecated_alias, getLogger, initLogging # # Models diff --git a/src/ocrd/processor/__init__.py b/src/ocrd/processor/__init__.py index f01e2b3c91..e63a4ef31a 100644 --- a/src/ocrd/processor/__init__.py +++ b/src/ocrd/processor/__init__.py @@ -1,8 +1,2 @@ -from .base import ( - Processor, -) -from .helpers import ( - run_cli, - run_processor, - generate_processor_help -) +from .base import Processor +from .helpers import generate_processor_help, run_cli, run_processor diff --git a/src/ocrd/processor/base.py b/src/ocrd/processor/base.py index 38b7848a03..6f9f6fa19f 100644 --- a/src/ocrd/processor/base.py +++ b/src/ocrd/processor/base.py @@ -9,34 +9,31 @@ 'run_processor' ] -from warnings import warn -from pkg_resources import resource_filename -from os.path import exists -from shutil import copyfileobj +import io import json import os -from os import getcwd -from pathlib import Path import sys import tarfile -import io +from os import getcwd +from os.path import exists +from pathlib import Path +from shutil import copyfileobj +from warnings import warn + +from pkg_resources import resource_filename -from ocrd_utils import ( - VERSION as OCRD_VERSION, - MIMETYPE_PAGE, - MIME_TO_EXT, - getLogger, - initLogging, - list_resource_candidates, - pushd_popd, - list_all_resources, - get_processor_resource_types -) +from ocrd_models.ocrd_page import LabelsType, LabelType, MetadataItemType +from ocrd_utils import MIME_TO_EXT, MIMETYPE_PAGE +from ocrd_utils import VERSION as OCRD_VERSION +from ocrd_utils import (get_processor_resource_types, getLogger, initLogging, + list_all_resources, list_resource_candidates, + pushd_popd) from ocrd_validators import ParameterValidator -from ocrd_models.ocrd_page import MetadataItemType, LabelType, LabelsType # XXX imports must remain for backwards-compatibilty -from .helpers import run_cli, run_processor, generate_processor_help # pylint: disable=unused-import +from .helpers import (generate_processor_help, # pylint: disable=unused-import + run_cli, run_processor) + class Processor(): """ diff --git a/src/ocrd/processor/builtin/dummy_processor.py b/src/ocrd/processor/builtin/dummy_processor.py index c0371e2d0e..90a881cea1 100644 --- a/src/ocrd/processor/builtin/dummy_processor.py +++ b/src/ocrd/processor/builtin/dummy_processor.py @@ -1,21 +1,16 @@ # pylint: disable=missing-module-docstring,invalid-name -from os.path import join, basename -from pkg_resources import resource_string +from os.path import basename, join import click +from pkg_resources import resource_string from ocrd import Processor from ocrd.decorators import ocrd_cli_options, ocrd_cli_wrap_processor -from ocrd_models.ocrd_page import to_xml -from ocrd_utils import ( - getLogger, - assert_file_grp_cardinality, - make_file_id, - MIME_TO_EXT, - MIMETYPE_PAGE, - parse_json_string_with_comments -) from ocrd_modelfactory import page_from_file +from ocrd_models.ocrd_page import to_xml +from ocrd_utils import (MIME_TO_EXT, MIMETYPE_PAGE, + assert_file_grp_cardinality, getLogger, make_file_id, + parse_json_string_with_comments) OCRD_TOOL = parse_json_string_with_comments(resource_string(__name__, 'dummy/ocrd-tool.json').decode('utf8')) diff --git a/src/ocrd/processor/helpers.py b/src/ocrd/processor/helpers.py index 209fa5f67d..bd9913f586 100644 --- a/src/ocrd/processor/helpers.py +++ b/src/ocrd/processor/helpers.py @@ -1,18 +1,19 @@ """ Helper methods for running and documenting processors """ -from os import chdir, getcwd -from time import perf_counter, process_time -from functools import lru_cache -import json import inspect +import json +from functools import lru_cache +from os import chdir, getcwd from subprocess import run +from time import perf_counter, process_time from typing import List from click import wrap_text -from ocrd.workspace import Workspace -from ocrd_utils import freeze_args, getLogger, config, setOverrideLogLevel, getLevelName +from ocrd.workspace import Workspace +from ocrd_utils import (config, freeze_args, getLevelName, getLogger, + setOverrideLogLevel) __all__ = [ 'generate_processor_help', diff --git a/src/ocrd/resolver.py b/src/ocrd/resolver.py index 4b8fe6b217..b4665af932 100644 --- a/src/ocrd/resolver.py +++ b/src/ocrd/resolver.py @@ -1,25 +1,19 @@ -from tempfile import mkdtemp from pathlib import Path +from tempfile import mkdtemp from warnings import warn import requests from requests.adapters import HTTPAdapter, Retry from ocrd.constants import TMP_PREFIX -from ocrd_utils import ( - config, - DEFAULT_METS_BASENAME, - getLogger, - is_local_filename, - get_local_filename, - remove_non_path_from_url, - is_file_in_directory, - nth_url_segment -) from ocrd.workspace import Workspace from ocrd_models import OcrdMets from ocrd_models.constants import NAMESPACES as NS from ocrd_models.utils import handle_oai_response +from ocrd_utils import (DEFAULT_METS_BASENAME, config, get_local_filename, + getLogger, is_file_in_directory, is_local_filename, + nth_url_segment, remove_non_path_from_url) + class Resolver(): """ diff --git a/src/ocrd/resource_manager.py b/src/ocrd/resource_manager.py index c668028e9c..9055b0eb04 100644 --- a/src/ocrd/resource_manager.py +++ b/src/ocrd/resource_manager.py @@ -1,29 +1,33 @@ -from pathlib import Path -from os.path import join -from os import environ, listdir, getcwd, path, unlink -from shutil import copytree, rmtree, copy -from fnmatch import filter as apply_glob from datetime import datetime +from fnmatch import filter as apply_glob +from os import environ, getcwd, listdir, path, unlink +from os.path import join +from pathlib import Path +from shutil import copy, copytree, rmtree from tarfile import open as open_tarfile -from urllib.parse import urlparse, unquote +from urllib.parse import unquote, urlparse from zipfile import ZipFile import requests -from gdown.parse_url import parse_url as gparse_url -from gdown.download import get_url_from_gdrive_confirmation -from yaml import safe_load, safe_dump - # https://github.com/OCR-D/core/issues/867 # https://stackoverflow.com/questions/50900727/skip-converting-entities-while-loading-a-yaml-string-using-pyyaml import yaml.constructor +from gdown.download import get_url_from_gdrive_confirmation +from gdown.parse_url import parse_url as gparse_url +from yaml import safe_dump, safe_load + yaml.constructor.SafeConstructor.yaml_constructors[u'tag:yaml.org,2002:timestamp'] = \ yaml.constructor.SafeConstructor.yaml_constructors[u'tag:yaml.org,2002:str'] +from ocrd_utils import (EXT_TO_MIME, config, directory_size, get_moduledir, + getLogger, guess_media_type, nth_url_segment) +from ocrd_utils.os import (get_ocrd_tool_json, get_processor_resource_types, + list_all_resources, pushd_popd) from ocrd_validators import OcrdResourceListValidator -from ocrd_utils import getLogger, directory_size, get_moduledir, EXT_TO_MIME, nth_url_segment, guess_media_type, config -from ocrd_utils.os import get_processor_resource_types, list_all_resources, pushd_popd, get_ocrd_tool_json + from .constants import RESOURCE_LIST_FILENAME, RESOURCE_USER_LIST_COMMENT + class OcrdResourceManager(): """ diff --git a/src/ocrd/task_sequence.py b/src/ocrd/task_sequence.py index 6d38601867..9035937d76 100644 --- a/src/ocrd/task_sequence.py +++ b/src/ocrd/task_sequence.py @@ -1,14 +1,18 @@ import json +from distutils.spawn import \ + find_executable as which # pylint: disable=import-error,no-name-in-module from shlex import split as shlex_split -from distutils.spawn import find_executable as which # pylint: disable=import-error,no-name-in-module -from subprocess import run, PIPE +from subprocess import PIPE, run -from ocrd_utils import getLogger, parse_json_string_or_file, set_json_key_value_overrides, get_ocrd_tool_json # from collections import Counter from ocrd.processor.base import run_cli from ocrd.resolver import Resolver -from ocrd_validators import ParameterValidator, WorkspaceValidator from ocrd_models import ValidationReport +from ocrd_utils import (get_ocrd_tool_json, getLogger, + parse_json_string_or_file, + set_json_key_value_overrides) +from ocrd_validators import ParameterValidator, WorkspaceValidator + class ProcessorTask(): diff --git a/src/ocrd/workspace.py b/src/ocrd/workspace.py index 7772c54d79..b54889e6e0 100644 --- a/src/ocrd/workspace.py +++ b/src/ocrd/workspace.py @@ -1,49 +1,32 @@ import io -from os import makedirs, unlink, listdir, path +from contextlib import contextmanager +from os import listdir, makedirs, path, unlink from pathlib import Path -from shutil import move, copyfileobj from re import sub +from shutil import copyfileobj, move from tempfile import NamedTemporaryFile -from contextlib import contextmanager -from cv2 import COLOR_GRAY2BGR, COLOR_RGB2BGR, cvtColor -from PIL import Image import numpy as np -from deprecated.sphinx import deprecated import requests +from cv2 import COLOR_GRAY2BGR, COLOR_RGB2BGR, cvtColor +from deprecated.sphinx import deprecated +from PIL import Image -from ocrd_models import OcrdMets, OcrdFile -from ocrd_models.ocrd_page import parse, BorderType, to_xml from ocrd_modelfactory import exif_from_filename, page_from_file -from ocrd_utils import ( - atomic_write, - getLogger, - image_from_polygon, - coordinates_of_segment, - adjust_canvas_to_rotation, - adjust_canvas_to_transposition, - shift_coordinates, - rotate_coordinates, - transform_coordinates, - transpose_coordinates, - crop_image, - rotate_image, - transpose_image, - bbox_from_polygon, - polygon_from_points, - xywh_from_bbox, - pushd_popd, - is_local_filename, - deprecated_alias, - DEFAULT_METS_BASENAME, - MIME_TO_EXT, - MIME_TO_PIL, - MIMETYPE_PAGE, - REGEX_PREFIX -) +from ocrd_models import OcrdFile, OcrdMets +from ocrd_models.ocrd_page import BorderType, parse, to_xml +from ocrd_utils import (DEFAULT_METS_BASENAME, MIME_TO_EXT, MIME_TO_PIL, + MIMETYPE_PAGE, REGEX_PREFIX, adjust_canvas_to_rotation, + adjust_canvas_to_transposition, atomic_write, + bbox_from_polygon, coordinates_of_segment, crop_image, + deprecated_alias, getLogger, image_from_polygon, + is_local_filename, polygon_from_points, pushd_popd, + rotate_coordinates, rotate_image, shift_coordinates, + transform_coordinates, transpose_coordinates, + transpose_image, xywh_from_bbox) -from .workspace_backup import WorkspaceBackupManager from .mets_server import ClientSideOcrdMets +from .workspace_backup import WorkspaceBackupManager __all__ = ['Workspace'] diff --git a/src/ocrd/workspace_backup.py b/src/ocrd/workspace_backup.py index 6cc3f1530d..63e8d1d233 100644 --- a/src/ocrd/workspace_backup.py +++ b/src/ocrd/workspace_backup.py @@ -1,15 +1,16 @@ +import hashlib from datetime import datetime -from os import makedirs -from os.path import join, basename, getsize, abspath from glob import glob +from os import makedirs +from os.path import abspath, basename, getsize, join from shutil import copy -import hashlib from ocrd_models import OcrdMets -from ocrd_utils import getLogger, atomic_write, DEFAULT_METS_BASENAME +from ocrd_utils import DEFAULT_METS_BASENAME, atomic_write, getLogger from .constants import BACKUP_DIR + def _chksum(s): return hashlib.sha256(s).hexdigest() diff --git a/src/ocrd/workspace_bagger.py b/src/ocrd/workspace_bagger.py index f838a65894..6fbdfdc36c 100644 --- a/src/ocrd/workspace_bagger.py +++ b/src/ocrd/workspace_bagger.py @@ -1,28 +1,25 @@ -from datetime import datetime -from os import makedirs, chdir, walk -from os.path import join, isdir, basename as os_path_basename, exists, relpath -from pathlib import Path -from shutil import make_archive, rmtree, copyfile, move -from tempfile import mkdtemp, TemporaryDirectory import re -import tempfile import sys -from bagit import Bag, make_manifests, _load_tag_file, _make_tag_file, _make_tagmanifest_file # pylint: disable=no-name-in-module +import tempfile +from datetime import datetime from distutils.dir_util import copy_tree +from os import chdir, makedirs, walk +from os.path import basename as os_path_basename +from os.path import exists, isdir, join, relpath +from pathlib import Path +from shutil import copyfile, make_archive, move, rmtree +from tempfile import TemporaryDirectory, mkdtemp + +from bagit import (Bag, _load_tag_file, # pylint: disable=no-name-in-module + _make_tag_file, _make_tagmanifest_file, make_manifests) from pkg_resources import get_distribution -from ocrd_utils import ( - pushd_popd, - getLogger, - MIME_TO_EXT, - unzip_file_to_dir, - DEFAULT_METS_BASENAME, - MIMETYPE_PAGE, - VERSION, -) -from ocrd_validators.constants import BAGIT_TXT, TMP_BAGIT_PREFIX, OCRD_BAGIT_PROFILE_URL from ocrd_modelfactory import page_from_file from ocrd_models.ocrd_page import to_xml +from ocrd_utils import (DEFAULT_METS_BASENAME, MIME_TO_EXT, MIMETYPE_PAGE, + VERSION, getLogger, pushd_popd, unzip_file_to_dir) +from ocrd_validators.constants import (BAGIT_TXT, OCRD_BAGIT_PROFILE_URL, + TMP_BAGIT_PREFIX) from .workspace import Workspace diff --git a/src/ocrd_modelfactory/__init__.py b/src/ocrd_modelfactory/__init__.py index e345ee061e..3231778dca 100644 --- a/src/ocrd_modelfactory/__init__.py +++ b/src/ocrd_modelfactory/__init__.py @@ -5,17 +5,15 @@ """ from datetime import datetime from pathlib import Path -from yaml import safe_load, safe_dump -from PIL import Image from lxml import etree as ET +from PIL import Image +from yaml import safe_dump, safe_load -from ocrd_utils import VERSION, MIMETYPE_PAGE, guess_media_type -from ocrd_models import OcrdExif, OcrdFile, ClientSideOcrdFile -from ocrd_models.ocrd_page import ( - PcGtsType, PageType, MetadataType, - parse, parseEtree -) +from ocrd_models import ClientSideOcrdFile, OcrdExif, OcrdFile +from ocrd_models.ocrd_page import (MetadataType, PageType, PcGtsType, parse, + parseEtree) +from ocrd_utils import MIMETYPE_PAGE, VERSION, guess_media_type __all__ = [ 'exif_from_filename', diff --git a/src/ocrd_models/__init__.py b/src/ocrd_models/__init__.py index a89ee1dec8..408c1c4309 100644 --- a/src/ocrd_models/__init__.py +++ b/src/ocrd_models/__init__.py @@ -1,9 +1,9 @@ """ APIs and schemas for various file formats in the OCR domain. """ -from .ocrd_agent import OcrdAgent, ClientSideOcrdAgent +from .ocrd_agent import ClientSideOcrdAgent, OcrdAgent from .ocrd_exif import OcrdExif -from .ocrd_file import OcrdFile, ClientSideOcrdFile +from .ocrd_file import ClientSideOcrdFile, OcrdFile from .ocrd_mets import OcrdMets from .ocrd_xml_base import OcrdXmlDocument from .report import ValidationReport diff --git a/src/ocrd_models/constants.py b/src/ocrd_models/constants.py index 6c8b0e1017..b1f34cd97a 100644 --- a/src/ocrd_models/constants.py +++ b/src/ocrd_models/constants.py @@ -1,9 +1,10 @@ """ Constants for ocrd_models. """ -from pkg_resources import resource_string import re +from pkg_resources import resource_string + __all__ = [ 'IDENTIFIER_PRIORITY', 'METS_XML_EMPTY', diff --git a/src/ocrd_models/ocrd_agent.py b/src/ocrd_models/ocrd_agent.py index 5a70e5b76a..51d4c8403b 100644 --- a/src/ocrd_models/ocrd_agent.py +++ b/src/ocrd_models/ocrd_agent.py @@ -2,9 +2,11 @@ API to ``mets:agent`` """ # import os -from .constants import NAMESPACES as NS, TAG_METS_AGENT, TAG_METS_NAME, TAG_METS_NOTE +from .constants import NAMESPACES as NS +from .constants import TAG_METS_AGENT, TAG_METS_NAME, TAG_METS_NOTE from .ocrd_xml_base import ET + class OcrdAgent(): """ Represents a diff --git a/src/ocrd_models/ocrd_exif.py b/src/ocrd_models/ocrd_exif.py index 598c499eff..fa017821c2 100644 --- a/src/ocrd_models/ocrd_exif.py +++ b/src/ocrd_models/ocrd_exif.py @@ -2,12 +2,14 @@ Technical image metadata """ -from math import sqrt -from io import BytesIO -from subprocess import run, PIPE from distutils.spawn import find_executable as which +from io import BytesIO +from math import sqrt +from subprocess import PIPE, run + from ocrd_utils import getLogger + class OcrdExif(): """Represents technical image metadata. diff --git a/src/ocrd_models/ocrd_file.py b/src/ocrd_models/ocrd_file.py index e8205a33d9..d15e1bf974 100644 --- a/src/ocrd_models/ocrd_file.py +++ b/src/ocrd_models/ocrd_file.py @@ -1,13 +1,15 @@ """ API to ``mets:file`` """ -from os.path import splitext, basename +from os.path import basename, splitext from pathlib import Path from ocrd_utils import deprecation_warning +from .constants import NAMESPACES as NS +from .constants import TAG_METS_FILE, TAG_METS_FLOCAT from .ocrd_xml_base import ET -from .constants import NAMESPACES as NS, TAG_METS_FLOCAT, TAG_METS_FILE + class OcrdFile(): """ diff --git a/src/ocrd_models/ocrd_mets.py b/src/ocrd_models/ocrd_mets.py index 3319f8f6ff..fbcdbb2d40 100644 --- a/src/ocrd_models/ocrd_mets.py +++ b/src/ocrd_models/ocrd_mets.py @@ -1,42 +1,27 @@ """ API to METS """ -from datetime import datetime import re import typing -from lxml import etree as ET from copy import deepcopy +from datetime import datetime from warnings import warn -from ocrd_utils import ( - getLogger, - deprecation_warning, - generate_range, - VERSION, - REGEX_PREFIX, - REGEX_FILE_ID -) +from lxml import etree as ET +from ocrd_utils import (REGEX_FILE_ID, REGEX_PREFIX, VERSION, + deprecation_warning, generate_range, getLogger) from ocrd_utils.config import config -from .constants import ( - NAMESPACES as NS, - TAG_METS_AGENT, - TAG_METS_DIV, - TAG_METS_FILE, - TAG_METS_FILEGRP, - TAG_METS_FILESEC, - TAG_METS_FPTR, - TAG_METS_METSHDR, - TAG_METS_STRUCTMAP, - IDENTIFIER_PRIORITY, - TAG_MODS_IDENTIFIER, - METS_XML_EMPTY, -) - -from .ocrd_xml_base import OcrdXmlDocument, ET -from .ocrd_file import OcrdFile +from .constants import IDENTIFIER_PRIORITY, METS_XML_EMPTY +from .constants import NAMESPACES as NS +from .constants import (TAG_METS_AGENT, TAG_METS_DIV, TAG_METS_FILE, + TAG_METS_FILEGRP, TAG_METS_FILESEC, TAG_METS_FPTR, + TAG_METS_METSHDR, TAG_METS_STRUCTMAP, + TAG_MODS_IDENTIFIER) from .ocrd_agent import OcrdAgent +from .ocrd_file import OcrdFile +from .ocrd_xml_base import ET, OcrdXmlDocument REGEX_PREFIX_LEN = len(REGEX_PREFIX) diff --git a/src/ocrd_models/ocrd_page.py b/src/ocrd_models/ocrd_page.py index b0cc2b3311..9205811b91 100644 --- a/src/ocrd_models/ocrd_page.py +++ b/src/ocrd_models/ocrd_page.py @@ -67,68 +67,33 @@ 'to_xml' ] -from .ocrd_page_generateds import ( - parse, - parseEtree, - parseString, - - AdvertRegionType, - AlternativeImageType, - BaselineType, - BorderType, - ChartRegionType, - ChemRegionType, - CoordsType, - CustomRegionType, - GlyphType, - GraphemeBaseType, - GraphemeGroupType, - GraphemeType, - GraphemesType, - GraphicRegionType, - GridPointsType, - GridType, - ImageRegionType, - LabelType, - LabelsType, - LayerType, - LayersType, - LineDrawingRegionType, - MapRegionType, - MathsRegionType, - MetadataItemType, - MetadataType, - MusicRegionType, - NoiseRegionType, - NonPrintingCharType, - OrderedGroupIndexedType, - OrderedGroupType, - PageType, - PcGtsType, - PrintSpaceType, - ReadingOrderType, - RegionRefIndexedType, - RegionRefType, - RegionType, - RelationType, - RelationsType, - RolesType, - SeparatorRegionType, - TableCellRoleType, - TableRegionType, - TextEquivType, - TextLineType, - TextRegionType, - TextStyleType, - UnknownRegionType, - UnorderedGroupIndexedType, - UnorderedGroupType, - UserAttributeType, - UserDefinedType, - WordType -) - from .constants import NAMESPACES +from .ocrd_page_generateds import (AdvertRegionType, AlternativeImageType, + BaselineType, BorderType, ChartRegionType, + ChemRegionType, CoordsType, + CustomRegionType, GlyphType, + GraphemeBaseType, GraphemeGroupType, + GraphemesType, GraphemeType, + GraphicRegionType, GridPointsType, GridType, + ImageRegionType, LabelsType, LabelType, + LayersType, LayerType, + LineDrawingRegionType, MapRegionType, + MathsRegionType, MetadataItemType, + MetadataType, MusicRegionType, + NoiseRegionType, NonPrintingCharType, + OrderedGroupIndexedType, OrderedGroupType, + PageType, PcGtsType, PrintSpaceType, + ReadingOrderType, RegionRefIndexedType, + RegionRefType, RegionType, RelationsType, + RelationType, RolesType, + SeparatorRegionType, TableCellRoleType, + TableRegionType, TextEquivType, + TextLineType, TextRegionType, TextStyleType, + UnknownRegionType, + UnorderedGroupIndexedType, + UnorderedGroupType, UserAttributeType, + UserDefinedType, WordType, parse, + parseEtree, parseString) # add docstrings parse.__doc__ = ( diff --git a/src/ocrd_models/ocrd_page_generateds.py b/src/ocrd_models/ocrd_page_generateds.py index d95ea4b321..5365469366 100644 --- a/src/ocrd_models/ocrd_page_generateds.py +++ b/src/ocrd_models/ocrd_page_generateds.py @@ -24,13 +24,14 @@ # core # -from itertools import zip_longest -import os -import sys -import re as re_ import base64 import datetime as datetime_ import decimal as decimal_ +import os +import re as re_ +import sys +from itertools import zip_longest + try: from lxml import etree as etree_ except ImportError: @@ -111,11 +112,13 @@ def parsexmlstring_(instring, parser=None, **kwargs): # try: - from generatedsnamespaces import GenerateDSNamespaceDefs as GenerateDSNamespaceDefs_ + from generatedsnamespaces import \ + GenerateDSNamespaceDefs as GenerateDSNamespaceDefs_ except ImportError: GenerateDSNamespaceDefs_ = {} try: - from generatedsnamespaces import GenerateDSNamespaceTypePrefixes as GenerateDSNamespaceTypePrefixes_ + from generatedsnamespaces import \ + GenerateDSNamespaceTypePrefixes as GenerateDSNamespaceTypePrefixes_ except ImportError: GenerateDSNamespaceTypePrefixes_ = {} @@ -1245,8 +1248,10 @@ def get_AllAlternativeImagePaths(self, page=True, region=True, line=True, word=T Returns: a list of image filename strings """ - from .constants import NAMESPACES, PAGE_REGION_TYPES # pylint: disable=relative-beyond-top-level,import-outside-toplevel from io import StringIO # pylint: disable=import-outside-toplevel + + from .constants import ( # pylint: disable=relative-beyond-top-level,import-outside-toplevel + NAMESPACES, PAGE_REGION_TYPES) ret = [] # XXX Since we're only interested in the **paths** of the images, # export, parse and xpath are less convoluted than traversing @@ -3122,7 +3127,8 @@ def _region_class(self, x): # pylint: disable=unused-argument return x.__class__.__name__.replace('RegionType', '') def _get_recursive_regions(self, regions, level, classes=None): - from .constants import PAGE_REGION_TYPES # pylint: disable=relative-beyond-top-level,import-outside-toplevel + from .constants import \ + PAGE_REGION_TYPES # pylint: disable=relative-beyond-top-level,import-outside-toplevel if level == 1: # stop recursion, filter classes if classes: diff --git a/src/ocrd_models/ocrd_xml_base.py b/src/ocrd_models/ocrd_xml_base.py index 7faefbad99..07e28e0485 100644 --- a/src/ocrd_models/ocrd_xml_base.py +++ b/src/ocrd_models/ocrd_xml_base.py @@ -2,12 +2,12 @@ Base class for XML documents loaded from either content or filename. """ from os.path import exists + from lxml import etree as ET from .constants import NAMESPACES from .utils import xmllint_format - for curie in NAMESPACES: ET.register_namespace(curie, NAMESPACES[curie]) diff --git a/src/ocrd_models/utils.py b/src/ocrd_models/utils.py index 1042f705c4..f7546c6d12 100644 --- a/src/ocrd_models/utils.py +++ b/src/ocrd_models/utils.py @@ -4,6 +4,7 @@ from lxml import etree as ET from ocrd_utils import getLogger + from .constants import NAMESPACES as NS __all__ = [ diff --git a/src/ocrd_network/__init__.py b/src/ocrd_network/__init__.py index 08153e5943..4e16f1092b 100644 --- a/src/ocrd_network/__init__.py +++ b/src/ocrd_network/__init__.py @@ -1,10 +1,7 @@ from .client import Client from .constants import NETWORK_AGENT_SERVER, NETWORK_AGENT_WORKER +from .param_validators import (DatabaseParamType, QueueServerParamType, + ServerAddressParamType) from .processing_server import ProcessingServer from .processing_worker import ProcessingWorker from .processor_server import ProcessorServer -from .param_validators import ( - DatabaseParamType, - ServerAddressParamType, - QueueServerParamType -) diff --git a/src/ocrd_network/cli/client.py b/src/ocrd_network/cli/client.py index 5c62ac44e8..36a039d486 100644 --- a/src/ocrd_network/cli/client.py +++ b/src/ocrd_network/cli/client.py @@ -1,13 +1,10 @@ -import click from typing import Optional -from ocrd_utils import DEFAULT_METS_BASENAME +import click -from ocrd.decorators import ( - parameter_option, - parameter_override_option -) +from ocrd.decorators import parameter_option, parameter_override_option from ocrd_network import Client +from ocrd_utils import DEFAULT_METS_BASENAME @click.group('client') diff --git a/src/ocrd_network/cli/processing_server.py b/src/ocrd_network/cli/processing_server.py index cf2aacab4a..e05759b24d 100644 --- a/src/ocrd_network/cli/processing_server.py +++ b/src/ocrd_network/cli/processing_server.py @@ -1,8 +1,6 @@ import click -from .. import ( - ProcessingServer, - ServerAddressParamType -) + +from .. import ProcessingServer, ServerAddressParamType @click.command('processing-server') diff --git a/src/ocrd_network/cli/processing_worker.py b/src/ocrd_network/cli/processing_worker.py index b626e8b55f..342d6812b5 100644 --- a/src/ocrd_network/cli/processing_worker.py +++ b/src/ocrd_network/cli/processing_worker.py @@ -1,11 +1,8 @@ import click + from ocrd_utils import get_ocrd_tool_json -from .. import ( - DatabaseParamType, - ProcessingWorker, - QueueServerParamType -) +from .. import DatabaseParamType, ProcessingWorker, QueueServerParamType @click.command('processing-worker') diff --git a/src/ocrd_network/cli/processor_server.py b/src/ocrd_network/cli/processor_server.py index 534a9a0fed..b133b265c1 100644 --- a/src/ocrd_network/cli/processor_server.py +++ b/src/ocrd_network/cli/processor_server.py @@ -1,9 +1,6 @@ import click -from .. import ( - DatabaseParamType, - ProcessorServer, - ServerAddressParamType -) + +from .. import DatabaseParamType, ProcessorServer, ServerAddressParamType @click.command('processor-server') diff --git a/src/ocrd_network/client.py b/src/ocrd_network/client.py index df04c3c27b..20afa95c2c 100644 --- a/src/ocrd_network/client.py +++ b/src/ocrd_network/client.py @@ -1,4 +1,5 @@ import json + import requests from ocrd_utils import config diff --git a/src/ocrd_network/database.py b/src/ocrd_network/database.py index 4945c13353..856a94ac3e 100644 --- a/src/ocrd_network/database.py +++ b/src/ocrd_network/database.py @@ -12,19 +12,16 @@ XXX: Currently the information is not preserved after the processing-server shuts down as the database (runs in docker) currently has no volume set. """ +from pathlib import Path +from typing import List +from uuid import uuid4 + from beanie import init_beanie from beanie.operators import In from motor.motor_asyncio import AsyncIOMotorClient -from uuid import uuid4 -from pathlib import Path -from typing import List -from .models import ( - DBProcessorJob, - DBWorkflowJob, - DBWorkspace, - DBWorkflowScript, -) +from .models import (DBProcessorJob, DBWorkflowJob, DBWorkflowScript, + DBWorkspace) from .utils import call_sync diff --git a/src/ocrd_network/deployer.py b/src/ocrd_network/deployer.py index ff54e0578a..c89f9b7108 100644 --- a/src/ocrd_network/deployer.py +++ b/src/ocrd_network/deployer.py @@ -7,34 +7,24 @@ Each Processing Worker is an instance of an OCR-D processor. """ from __future__ import annotations -from typing import Dict, List, Union -from re import search as re_search -from pathlib import Path + import subprocess +from pathlib import Path +from re import search as re_search from time import sleep +from typing import Dict, List, Union from ocrd_utils import config, getLogger, safe_filename from .constants import NETWORK_AGENT_SERVER, NETWORK_AGENT_WORKER -from .deployment_utils import ( - create_docker_client, - DeployType, - verify_mongodb_available, - verify_rabbitmq_available, -) +from .deployment_utils import (DeployType, create_docker_client, + verify_mongodb_available, + verify_rabbitmq_available) from .logging import get_mets_server_logging_file_path -from .runtime_data import ( - DataHost, - DataMongoDB, - DataProcessingWorker, - DataProcessorServer, - DataRabbitMQ -) -from .utils import ( - is_mets_server_running, - stop_mets_server, - validate_and_load_config -) +from .runtime_data import (DataHost, DataMongoDB, DataProcessingWorker, + DataProcessorServer, DataRabbitMQ) +from .utils import (is_mets_server_running, stop_mets_server, + validate_and_load_config) class Deployer: diff --git a/src/ocrd_network/deployment_utils.py b/src/ocrd_network/deployment_utils.py index 8c3ff7e46c..4210a62902 100644 --- a/src/ocrd_network/deployment_utils.py +++ b/src/ocrd_network/deployment_utils.py @@ -1,13 +1,15 @@ from __future__ import annotations + +import re from enum import Enum +from time import sleep + from docker import APIClient, DockerClient from docker.transport import SSHHTTPAdapter from paramiko import AutoAddPolicy, SSHClient -from time import sleep -import re +from pymongo import MongoClient from .rabbitmq_utils import RMQPublisher -from pymongo import MongoClient __all__ = [ 'create_docker_client', diff --git a/src/ocrd_network/logging.py b/src/ocrd_network/logging.py index 3365e2ddcc..20a04d3f5b 100644 --- a/src/ocrd_network/logging.py +++ b/src/ocrd_network/logging.py @@ -1,5 +1,6 @@ from pathlib import Path -from ocrd_utils import safe_filename, config + +from ocrd_utils import config, safe_filename from .constants import NETWORK_AGENT_SERVER, NETWORK_AGENT_WORKER diff --git a/src/ocrd_network/models/__init__.py b/src/ocrd_network/models/__init__.py index dc8231f767..708bc9c3dd 100644 --- a/src/ocrd_network/models/__init__.py +++ b/src/ocrd_network/models/__init__.py @@ -16,15 +16,9 @@ 'StateEnum', ] -from .job import ( - DBProcessorJob, - DBWorkflowJob, - PYJobInput, - PYJobOutput, - PYWorkflowJobOutput, - StateEnum -) +from .job import (DBProcessorJob, DBWorkflowJob, PYJobInput, PYJobOutput, + PYWorkflowJobOutput, StateEnum) from .messages import PYResultMessage from .ocrd_tool import PYOcrdTool -from .workspace import DBWorkspace from .workflow import DBWorkflowScript +from .workspace import DBWorkspace diff --git a/src/ocrd_network/models/job.py b/src/ocrd_network/models/job.py index e5230aa5fd..891fe1ed6f 100644 --- a/src/ocrd_network/models/job.py +++ b/src/ocrd_network/models/job.py @@ -1,9 +1,10 @@ -from beanie import Document from datetime import datetime from enum import Enum -from pydantic import BaseModel from typing import Dict, List, Optional +from beanie import Document +from pydantic import BaseModel + from ocrd_network import NETWORK_AGENT_WORKER diff --git a/src/ocrd_network/models/messages.py b/src/ocrd_network/models/messages.py index 062f1a9a24..15ca7c57e1 100644 --- a/src/ocrd_network/models/messages.py +++ b/src/ocrd_network/models/messages.py @@ -1,5 +1,7 @@ -from pydantic import BaseModel from typing import Optional + +from pydantic import BaseModel + from .job import StateEnum diff --git a/src/ocrd_network/models/ocrd_tool.py b/src/ocrd_network/models/ocrd_tool.py index b3e2ceaea8..ce6b96e6a7 100644 --- a/src/ocrd_network/models/ocrd_tool.py +++ b/src/ocrd_network/models/ocrd_tool.py @@ -1,6 +1,7 @@ -from pydantic import BaseModel from typing import List, Optional +from pydantic import BaseModel + class PYOcrdTool(BaseModel): executable: str diff --git a/src/ocrd_network/models/workspace.py b/src/ocrd_network/models/workspace.py index 670cb14b58..13bcf0251d 100644 --- a/src/ocrd_network/models/workspace.py +++ b/src/ocrd_network/models/workspace.py @@ -1,6 +1,7 @@ -from beanie import Document from typing import Optional +from beanie import Document + class DBWorkspace(Document): """ diff --git a/src/ocrd_network/param_validators.py b/src/ocrd_network/param_validators.py index 87cfeee722..1736f5320e 100644 --- a/src/ocrd_network/param_validators.py +++ b/src/ocrd_network/param_validators.py @@ -1,9 +1,6 @@ from click import ParamType -from .utils import ( - verify_database_uri, - verify_and_parse_mq_uri -) +from .utils import verify_and_parse_mq_uri, verify_database_uri class ServerAddressParamType(ParamType): diff --git a/src/ocrd_network/process_helpers.py b/src/ocrd_network/process_helpers.py index 04dcd17d00..fad1344439 100644 --- a/src/ocrd_network/process_helpers.py +++ b/src/ocrd_network/process_helpers.py @@ -1,9 +1,9 @@ -from contextlib import nullcontext import json +from contextlib import nullcontext from typing import List, Optional from ocrd.processor.helpers import run_cli, run_processor -from ocrd_utils import redirect_stderr_and_stdout_to_file, initLogging +from ocrd_utils import initLogging, redirect_stderr_and_stdout_to_file from .utils import get_ocrd_workspace_instance diff --git a/src/ocrd_network/processing_server.py b/src/ocrd_network/processing_server.py index e6606f90f0..4f6df6b8b1 100644 --- a/src/ocrd_network/processing_server.py +++ b/src/ocrd_network/processing_server.py @@ -1,71 +1,43 @@ -from hashlib import md5 -import httpx import json +from hashlib import md5 from logging import FileHandler, Formatter from os import getpid from pathlib import Path -import requests from typing import Dict, List, Union from urllib.parse import urljoin -import uvicorn -from fastapi import ( - FastAPI, - status, - Request, - HTTPException, - UploadFile, - File, -) +import httpx +import requests +import uvicorn +from fastapi import FastAPI, File, HTTPException, Request, UploadFile, status from fastapi.exceptions import RequestValidationError from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse from pika.exceptions import ChannelClosedByBroker from ocrd import Resolver, Workspace from ocrd.task_sequence import ProcessorTask -from ocrd_utils import initLogging, getLogger, LOG_FORMAT +from ocrd_utils import LOG_FORMAT, getLogger, initLogging from .constants import NETWORK_AGENT_SERVER, NETWORK_AGENT_WORKER -from .database import ( - initiate_database, - db_create_workspace, - db_get_processing_job, - db_get_processing_jobs, - db_get_workflow_job, - db_get_workspace, - db_update_processing_job, - db_update_workspace, - db_get_workflow_script, - db_find_first_workflow_script_by_content -) +from .database import (db_create_workspace, + db_find_first_workflow_script_by_content, + db_get_processing_job, db_get_processing_jobs, + db_get_workflow_job, db_get_workflow_script, + db_get_workspace, db_update_processing_job, + db_update_workspace, initiate_database) from .deployer import Deployer from .logging import get_processing_server_logging_file_path -from .models import ( - DBProcessorJob, - DBWorkflowJob, - DBWorkflowScript, - PYJobInput, - PYJobOutput, - PYResultMessage, - PYWorkflowJobOutput, - StateEnum -) -from .rabbitmq_utils import RMQPublisher, OcrdProcessingMessage +from .models import (DBProcessorJob, DBWorkflowJob, DBWorkflowScript, + PYJobInput, PYJobOutput, PYResultMessage, + PYWorkflowJobOutput, StateEnum) +from .rabbitmq_utils import OcrdProcessingMessage, RMQPublisher from .server_cache import CacheLockedPages, CacheProcessingRequests -from .server_utils import ( - _get_processor_job, - _get_processor_job_log, - expand_page_ids, - validate_and_return_mets_path, - validate_job_input -) -from .utils import ( - download_ocrd_all_tool_json, - generate_created_time, - generate_id, - get_ocrd_workspace_physical_pages, - validate_workflow, -) +from .server_utils import (_get_processor_job, _get_processor_job_log, + expand_page_ids, validate_and_return_mets_path, + validate_job_input) +from .utils import (download_ocrd_all_tool_json, generate_created_time, + generate_id, get_ocrd_workspace_physical_pages, + validate_workflow) class ProcessingServer(FastAPI): diff --git a/src/ocrd_network/processing_worker.py b/src/ocrd_network/processing_worker.py index ff4287c848..e2aa702cf5 100644 --- a/src/ocrd_network/processing_worker.py +++ b/src/ocrd_network/processing_worker.py @@ -12,34 +12,23 @@ from logging import FileHandler, Formatter from os import getpid from time import sleep -import pika.spec + import pika.adapters.blocking_connection +import pika.spec from pika.exceptions import AMQPConnectionError -from ocrd_utils import config, getLogger, LOG_FORMAT -from .database import ( - sync_initiate_database, - sync_db_get_workspace, - sync_db_update_processing_job, -) -from .logging import ( - get_processing_job_logging_file_path, - get_processing_worker_logging_file_path -) +from ocrd_utils import LOG_FORMAT, config, getLogger + +from .database import (sync_db_get_workspace, sync_db_update_processing_job, + sync_initiate_database) +from .logging import (get_processing_job_logging_file_path, + get_processing_worker_logging_file_path) from .models import StateEnum from .process_helpers import invoke_processor -from .rabbitmq_utils import ( - OcrdProcessingMessage, - OcrdResultMessage, - RMQConsumer, - RMQPublisher -) -from .utils import ( - calculate_execution_time, - post_to_callback_url, - verify_database_uri, - verify_and_parse_mq_uri -) +from .rabbitmq_utils import (OcrdProcessingMessage, OcrdResultMessage, + RMQConsumer, RMQPublisher) +from .utils import (calculate_execution_time, post_to_callback_url, + verify_and_parse_mq_uri, verify_database_uri) class ProcessingWorker: diff --git a/src/ocrd_network/processor_server.py b/src/ocrd_network/processor_server.py index cacba84615..8651995471 100644 --- a/src/ocrd_network/processor_server.py +++ b/src/ocrd_network/processor_server.py @@ -1,49 +1,25 @@ from datetime import datetime from logging import FileHandler, Formatter from os import getpid -from subprocess import run, PIPE -import uvicorn +from subprocess import PIPE, run +import uvicorn from fastapi import FastAPI, HTTPException, status from fastapi.responses import FileResponse -from ocrd_utils import ( - initLogging, - get_ocrd_tool_json, - getLogger, - LOG_FORMAT, - parse_json_string_with_comments -) -from .database import ( - DBProcessorJob, - db_get_workspace, - db_update_processing_job, - db_get_processing_job, - initiate_database -) -from .logging import ( - get_processor_server_logging_file_path, - get_processing_job_logging_file_path, -) -from .models import ( - PYJobInput, - PYJobOutput, - PYOcrdTool, - StateEnum -) +from ocrd_utils import (LOG_FORMAT, get_ocrd_tool_json, getLogger, initLogging, + parse_json_string_with_comments) + +from .database import (DBProcessorJob, db_get_processing_job, db_get_workspace, + db_update_processing_job, initiate_database) +from .logging import (get_processing_job_logging_file_path, + get_processor_server_logging_file_path) +from .models import PYJobInput, PYJobOutput, PYOcrdTool, StateEnum from .process_helpers import invoke_processor from .rabbitmq_utils import OcrdResultMessage -from .server_utils import ( - _get_processor_job, - _get_processor_job_log, - validate_and_return_mets_path, - validate_job_input -) -from .utils import ( - calculate_execution_time, - post_to_callback_url, - generate_id, -) +from .server_utils import (_get_processor_job, _get_processor_job_log, + validate_and_return_mets_path, validate_job_input) +from .utils import calculate_execution_time, generate_id, post_to_callback_url class ProcessorServer(FastAPI): diff --git a/src/ocrd_network/rabbitmq_utils/__init__.py b/src/ocrd_network/rabbitmq_utils/__init__.py index 2d5f55e62c..29fc8146cc 100644 --- a/src/ocrd_network/rabbitmq_utils/__init__.py +++ b/src/ocrd_network/rabbitmq_utils/__init__.py @@ -6,10 +6,7 @@ 'OcrdResultMessage' ] -from .consumer import RMQConsumer from .connector import RMQConnector +from .consumer import RMQConsumer +from .ocrd_messages import OcrdProcessingMessage, OcrdResultMessage from .publisher import RMQPublisher -from .ocrd_messages import ( - OcrdProcessingMessage, - OcrdResultMessage -) diff --git a/src/ocrd_network/rabbitmq_utils/connector.py b/src/ocrd_network/rabbitmq_utils/connector.py index 6dbc6ea0d3..056714ab07 100644 --- a/src/ocrd_network/rabbitmq_utils/connector.py +++ b/src/ocrd_network/rabbitmq_utils/connector.py @@ -4,18 +4,16 @@ RabbitMQ documentation. """ from typing import Any, Optional, Union -from pika import BasicProperties, BlockingConnection, ConnectionParameters, PlainCredentials + +from pika import (BasicProperties, BlockingConnection, ConnectionParameters, + PlainCredentials) from pika.adapters.blocking_connection import BlockingChannel -from .constants import ( - DEFAULT_EXCHANGER_NAME, - DEFAULT_EXCHANGER_TYPE, - DEFAULT_QUEUE, - DEFAULT_ROUTER, - RABBIT_MQ_HOST as HOST, - RABBIT_MQ_PORT as PORT, - RABBIT_MQ_VHOST as VHOST, - PREFETCH_COUNT -) + +from .constants import (DEFAULT_EXCHANGER_NAME, DEFAULT_EXCHANGER_TYPE, + DEFAULT_QUEUE, DEFAULT_ROUTER, PREFETCH_COUNT) +from .constants import RABBIT_MQ_HOST as HOST +from .constants import RABBIT_MQ_PORT as PORT +from .constants import RABBIT_MQ_VHOST as VHOST class RMQConnector: diff --git a/src/ocrd_network/rabbitmq_utils/consumer.py b/src/ocrd_network/rabbitmq_utils/consumer.py index 0d8d905eab..b7e1462121 100644 --- a/src/ocrd_network/rabbitmq_utils/consumer.py +++ b/src/ocrd_network/rabbitmq_utils/consumer.py @@ -4,15 +4,16 @@ RabbitMQ documentation. """ from typing import Any, Union + from pika import PlainCredentials + from ocrd_utils import getLogger -from .constants import ( - DEFAULT_QUEUE, - RABBIT_MQ_HOST as HOST, - RABBIT_MQ_PORT as PORT, - RABBIT_MQ_VHOST as VHOST -) + from .connector import RMQConnector +from .constants import DEFAULT_QUEUE +from .constants import RABBIT_MQ_HOST as HOST +from .constants import RABBIT_MQ_PORT as PORT +from .constants import RABBIT_MQ_VHOST as VHOST class RMQConsumer(RMQConnector): diff --git a/src/ocrd_network/rabbitmq_utils/ocrd_messages.py b/src/ocrd_network/rabbitmq_utils/ocrd_messages.py index 4016cd71bb..19a0a804a3 100644 --- a/src/ocrd_network/rabbitmq_utils/ocrd_messages.py +++ b/src/ocrd_network/rabbitmq_utils/ocrd_messages.py @@ -1,5 +1,7 @@ from __future__ import annotations + from typing import Any, Dict, List, Optional + import yaml from ocrd_validators import OcrdNetworkMessageValidator diff --git a/src/ocrd_network/rabbitmq_utils/publisher.py b/src/ocrd_network/rabbitmq_utils/publisher.py index f77975a7e0..261531b256 100644 --- a/src/ocrd_network/rabbitmq_utils/publisher.py +++ b/src/ocrd_network/rabbitmq_utils/publisher.py @@ -4,16 +4,16 @@ RabbitMQ documentation. """ from typing import Optional + from pika import BasicProperties, PlainCredentials + from ocrd_utils import getLogger -from .constants import ( - DEFAULT_EXCHANGER_NAME, - DEFAULT_ROUTER, - RABBIT_MQ_HOST as HOST, - RABBIT_MQ_PORT as PORT, - RABBIT_MQ_VHOST as VHOST -) + from .connector import RMQConnector +from .constants import DEFAULT_EXCHANGER_NAME, DEFAULT_ROUTER +from .constants import RABBIT_MQ_HOST as HOST +from .constants import RABBIT_MQ_PORT as PORT +from .constants import RABBIT_MQ_VHOST as VHOST class RMQPublisher(RMQConnector): diff --git a/src/ocrd_network/runtime_data.py b/src/ocrd_network/runtime_data.py index 59c658ada1..329ba0d934 100644 --- a/src/ocrd_network/runtime_data.py +++ b/src/ocrd_network/runtime_data.py @@ -1,11 +1,9 @@ from __future__ import annotations + from typing import Dict, List -from .deployment_utils import ( - create_docker_client, - create_ssh_client, - DeployType -) +from .deployment_utils import (DeployType, create_docker_client, + create_ssh_client) __all__ = [ 'DataHost', diff --git a/src/ocrd_network/server_cache.py b/src/ocrd_network/server_cache.py index 591ee1c8bb..f39c7fd0fa 100644 --- a/src/ocrd_network/server_cache.py +++ b/src/ocrd_network/server_cache.py @@ -1,13 +1,13 @@ from __future__ import annotations -from typing import Dict, List + from logging import FileHandler, Formatter +from typing import Dict, List + +from ocrd_utils import LOG_FORMAT, getLogger -from ocrd_utils import getLogger, LOG_FORMAT from .database import db_get_processing_job, db_update_processing_job -from .logging import ( - get_cache_locked_pages_logging_file_path, - get_cache_processing_requests_logging_file_path -) +from .logging import (get_cache_locked_pages_logging_file_path, + get_cache_processing_requests_logging_file_path) from .models import PYJobInput, StateEnum __all__ = [ diff --git a/src/ocrd_network/server_utils.py b/src/ocrd_network/server_utils.py index fd7c0f7966..fad9b1211b 100644 --- a/src/ocrd_network/server_utils.py +++ b/src/ocrd_network/server_utils.py @@ -1,17 +1,14 @@ import re -from fastapi import HTTPException, status -from fastapi.responses import FileResponse from pathlib import Path from typing import List + +from fastapi import HTTPException, status +from fastapi.responses import FileResponse + +from ocrd_utils import REGEX_PREFIX, generate_range from ocrd_validators import ParameterValidator -from ocrd_utils import ( - generate_range, - REGEX_PREFIX -) -from .database import ( - db_get_processing_job, - db_get_workspace, -) + +from .database import db_get_processing_job, db_get_workspace from .models import PYJobInput, PYJobOutput diff --git a/src/ocrd_network/utils.py b/src/ocrd_network/utils.py index 8f92706f46..02ecd6ba14 100644 --- a/src/ocrd_network/utils.py +++ b/src/ocrd_network/utils.py @@ -1,18 +1,20 @@ from datetime import datetime from functools import wraps +from re import match as re_match +from typing import Dict, List +from uuid import uuid4 + from pika import URLParameters from pymongo import uri_parser as mongo_uri_parser -from re import match as re_match from requests import Session as Session_TCP from requests_unixsocket import Session as Session_UDS -from typing import Dict, List -from uuid import uuid4 from yaml import safe_load from ocrd import Resolver, Workspace +from ocrd.task_sequence import ProcessorTask from ocrd_validators import ProcessingServerConfigValidator + from .rabbitmq_utils import OcrdResultMessage -from ocrd.task_sequence import ProcessorTask # Based on: https://gist.github.com/phizaz/20c36c6734878c6ec053245a477572ec diff --git a/src/ocrd_page_user_methods.py b/src/ocrd_page_user_methods.py index 8a2332e6e5..dcf836cc71 100644 --- a/src/ocrd_page_user_methods.py +++ b/src/ocrd_page_user_methods.py @@ -2,10 +2,11 @@ # source: https://bitbucket.org/dkuhlman/generateds/src/default/gends_user_methods.py -import re import codecs +import re from os.path import dirname, join + # # You must include the following class definition at the top of # your method specification file. diff --git a/src/ocrd_page_user_methods/get_AllAlternativeImagePaths.py b/src/ocrd_page_user_methods/get_AllAlternativeImagePaths.py index c8897676e7..5b0ef40262 100644 --- a/src/ocrd_page_user_methods/get_AllAlternativeImagePaths.py +++ b/src/ocrd_page_user_methods/get_AllAlternativeImagePaths.py @@ -12,8 +12,10 @@ def get_AllAlternativeImagePaths(self, page=True, region=True, line=True, word=T Returns: a list of image filename strings """ - from .constants import NAMESPACES, PAGE_REGION_TYPES # pylint: disable=relative-beyond-top-level,import-outside-toplevel from io import StringIO # pylint: disable=import-outside-toplevel + + from .constants import ( # pylint: disable=relative-beyond-top-level,import-outside-toplevel + NAMESPACES, PAGE_REGION_TYPES) ret = [] # XXX Since we're only interested in the **paths** of the images, # export, parse and xpath are less convoluted than traversing diff --git a/src/ocrd_page_user_methods/get_AllRegions.py b/src/ocrd_page_user_methods/get_AllRegions.py index e791cb489a..f14746389c 100644 --- a/src/ocrd_page_user_methods/get_AllRegions.py +++ b/src/ocrd_page_user_methods/get_AllRegions.py @@ -3,7 +3,8 @@ def _region_class(self, x): # pylint: disable=unused-argument return x.__class__.__name__.replace('RegionType', '') def _get_recursive_regions(self, regions, level, classes=None): - from .constants import PAGE_REGION_TYPES # pylint: disable=relative-beyond-top-level,import-outside-toplevel + from .constants import \ + PAGE_REGION_TYPES # pylint: disable=relative-beyond-top-level,import-outside-toplevel if level == 1: # stop recursion, filter classes if classes: diff --git a/src/ocrd_utils/__init__.py b/src/ocrd_utils/__init__.py index 90cd554779..6f5171c78e 100644 --- a/src/ocrd_utils/__init__.py +++ b/src/ocrd_utils/__init__.py @@ -105,100 +105,33 @@ Decorator to mark a kwarg as deprecated """ -from .constants import ( - DEFAULT_METS_BASENAME, - EXT_TO_MIME, - MIMETYPE_PAGE, - MIME_TO_EXT, - MIME_TO_PIL, - PIL_TO_MIME, - REGEX_PREFIX, - REGEX_FILE_ID, - RESOURCE_LOCATIONS, - LOG_FORMAT, - LOG_TIMEFMT, - VERSION, - ) - -from .deprecate import ( - deprecated_alias, - rename_kwargs, - deprecation_warning) - -from .image import ( - adjust_canvas_to_rotation, - adjust_canvas_to_transposition, - bbox_from_points, - bbox_from_polygon, - bbox_from_xywh, - coordinates_for_segment, - coordinates_of_segment, - crop_image, - image_from_polygon, - points_from_bbox, - points_from_polygon, - points_from_x0y0x1y1, - points_from_xywh, - points_from_y0x0y1x1, - polygon_from_bbox, - polygon_from_points, - polygon_from_x0y0x1y1, - polygon_from_xywh, - polygon_mask, - rotate_coordinates, - rotate_image, - shift_coordinates, - transform_coordinates, - transpose_coordinates, - transpose_image, - xywh_from_bbox, - xywh_from_points, - xywh_from_polygon) - -from .introspect import ( - freeze_args, - set_json_key_value_overrides, - membername -) - -from .logging import ( - tf_disable_interactive_logs, - disableLogging, - getLevelName, - getLogger, - initLogging, - setOverrideLogLevel, - ) - -from .os import ( - abspath, - directory_size, - get_processor_resource_types, - get_ocrd_tool_json, - get_moduledir, - guess_media_type, - list_all_resources, - is_file_in_directory, - list_resource_candidates, - atomic_write, - pushd_popd, - unzip_file_to_dir, - redirect_stderr_and_stdout_to_file, - ) - -from .str import ( - assert_file_grp_cardinality, - concat_padded, - generate_range, - get_local_filename, - is_local_filename, - is_string, - make_file_id, - nth_url_segment, - partition_list, - parse_json_string_or_file, - parse_json_string_with_comments, - remove_non_path_from_url, - safe_filename) - from .config import config +from .constants import (DEFAULT_METS_BASENAME, EXT_TO_MIME, LOG_FORMAT, + LOG_TIMEFMT, MIME_TO_EXT, MIME_TO_PIL, MIMETYPE_PAGE, + PIL_TO_MIME, REGEX_FILE_ID, REGEX_PREFIX, + RESOURCE_LOCATIONS, VERSION) +from .deprecate import deprecated_alias, deprecation_warning, rename_kwargs +from .image import (adjust_canvas_to_rotation, adjust_canvas_to_transposition, + bbox_from_points, bbox_from_polygon, bbox_from_xywh, + coordinates_for_segment, coordinates_of_segment, + crop_image, image_from_polygon, points_from_bbox, + points_from_polygon, points_from_x0y0x1y1, + points_from_xywh, points_from_y0x0y1x1, polygon_from_bbox, + polygon_from_points, polygon_from_x0y0x1y1, + polygon_from_xywh, polygon_mask, rotate_coordinates, + rotate_image, shift_coordinates, transform_coordinates, + transpose_coordinates, transpose_image, xywh_from_bbox, + xywh_from_points, xywh_from_polygon) +from .introspect import freeze_args, membername, set_json_key_value_overrides +from .logging import (disableLogging, getLevelName, getLogger, initLogging, + setOverrideLogLevel, tf_disable_interactive_logs) +from .os import (abspath, atomic_write, directory_size, get_moduledir, + get_ocrd_tool_json, get_processor_resource_types, + guess_media_type, is_file_in_directory, list_all_resources, + list_resource_candidates, pushd_popd, + redirect_stderr_and_stdout_to_file, unzip_file_to_dir) +from .str import (assert_file_grp_cardinality, concat_padded, generate_range, + get_local_filename, is_local_filename, is_string, + make_file_id, nth_url_segment, parse_json_string_or_file, + parse_json_string_with_comments, partition_list, + remove_non_path_from_url, safe_filename) diff --git a/src/ocrd_utils/constants.py b/src/ocrd_utils/constants.py index 0b9f0ae02b..5d4fdda30d 100644 --- a/src/ocrd_utils/constants.py +++ b/src/ocrd_utils/constants.py @@ -1,9 +1,10 @@ """ Constants for ocrd_utils. """ -from pkg_resources import get_distribution from re import compile as regex_compile +from pkg_resources import get_distribution + __all__ = [ 'EXT_TO_MIME', 'LOG_FORMAT', diff --git a/src/ocrd_utils/deprecate.py b/src/ocrd_utils/deprecate.py index d17efeb58d..4876e7bb68 100644 --- a/src/ocrd_utils/deprecate.py +++ b/src/ocrd_utils/deprecate.py @@ -1,6 +1,7 @@ import functools import warnings + def deprecation_warning(msg, stacklevel=2): warnings.warn(msg, DeprecationWarning, stacklevel) diff --git a/src/ocrd_utils/image.py b/src/ocrd_utils/image.py index 3bc14e6612..12e74c8bc3 100644 --- a/src/ocrd_utils/image.py +++ b/src/ocrd_utils/image.py @@ -1,10 +1,10 @@ import sys import numpy as np -from PIL import Image, ImageStat, ImageDraw, ImageChops +from PIL import Image, ImageChops, ImageDraw, ImageStat -from .logging import getLogger from .introspect import membername +from .logging import getLogger # Allow processing of images with up to 1.6bn pixels # https://github.com/OCR-D/core/issues/735 diff --git a/src/ocrd_utils/introspect.py b/src/ocrd_utils/introspect.py index cfd3d32b52..51d524271e 100644 --- a/src/ocrd_utils/introspect.py +++ b/src/ocrd_utils/introspect.py @@ -3,6 +3,7 @@ """ import json from functools import wraps + from frozendict import frozendict diff --git a/src/ocrd_utils/logging.py b/src/ocrd_utils/logging.py index 7311f2b01d..a135aefa70 100644 --- a/src/ocrd_utils/logging.py +++ b/src/ocrd_utils/logging.py @@ -28,16 +28,14 @@ from __future__ import absolute_import -from traceback import format_stack - import logging import logging.config -from pathlib import Path import sys +from pathlib import Path +from traceback import format_stack -from .constants import LOG_FORMAT, LOG_TIMEFMT from .config import config - +from .constants import LOG_FORMAT, LOG_TIMEFMT __all__ = [ 'disableLogging', @@ -82,9 +80,11 @@ def tf_disable_interactive_logs(): try: from os import environ + # This env variable must be set before importing from Keras environ['TF_CPP_MIN_LOG_LEVEL'] = '3' from tensorflow.keras.utils import disable_interactive_logging + # Enabled interactive logging throws an exception # due to a call of sys.stdout.flush() disable_interactive_logging() diff --git a/src/ocrd_utils/os.py b/src/ocrd_utils/os.py index a416ccb12e..4339deb3d4 100644 --- a/src/ocrd_utils/os.py +++ b/src/ocrd_utils/os.py @@ -15,26 +15,29 @@ 'redirect_stderr_and_stdout_to_file', ] -from tempfile import TemporaryDirectory, gettempdir -from functools import lru_cache from contextlib import contextmanager, redirect_stderr, redirect_stdout from distutils.spawn import find_executable as which +from functools import lru_cache from json import loads from json.decoder import JSONDecodeError -from os import getcwd, chdir, stat, chmod, umask, environ +from mimetypes import guess_type as mimetypes_guess +from os import chdir, chmod, environ, getcwd, stat, umask +from os.path import abspath as abspath_ +from os.path import exists, isdir, join from pathlib import Path -from os.path import exists, abspath as abspath_, join, isdir +from subprocess import PIPE, run +from tempfile import TemporaryDirectory, gettempdir from zipfile import ZipFile -from subprocess import run, PIPE -from mimetypes import guess_type as mimetypes_guess -from filetype import guess as filetype_guess -from atomicwrites import atomic_write as atomic_write_, AtomicWriter +from atomicwrites import AtomicWriter +from atomicwrites import atomic_write as atomic_write_ +from filetype import guess as filetype_guess -from .constants import EXT_TO_MIME from .config import config +from .constants import EXT_TO_MIME from .logging import getLogger + def abspath(url): """ Get a full path to a file or file URL diff --git a/src/ocrd_utils/str.py b/src/ocrd_utils/str.py index f5b9242d35..29cf55fedd 100644 --- a/src/ocrd_utils/str.py +++ b/src/ocrd_utils/str.py @@ -2,13 +2,15 @@ Utility functions for strings, paths and URL. """ -import re import json -from .constants import REGEX_FILE_ID -from .deprecate import deprecation_warning +import re from warnings import warn + from numpy import array_split +from .constants import REGEX_FILE_ID +from .deprecate import deprecation_warning + __all__ = [ 'assert_file_grp_cardinality', 'concat_padded', diff --git a/src/ocrd_validators/__init__.py b/src/ocrd_validators/__init__.py index 48e5b30461..d0ec64a93a 100644 --- a/src/ocrd_validators/__init__.py +++ b/src/ocrd_validators/__init__.py @@ -15,14 +15,14 @@ 'OcrdNetworkMessageValidator' ] -from .parameter_validator import ParameterValidator -from .workspace_validator import WorkspaceValidator -from .page_validator import PageValidator +from .ocrd_network_message_validator import OcrdNetworkMessageValidator from .ocrd_tool_validator import OcrdToolValidator -from .resource_list_validator import OcrdResourceListValidator from .ocrd_zip_validator import OcrdZipValidator -from .xsd_validator import XsdValidator +from .page_validator import PageValidator +from .parameter_validator import ParameterValidator +from .processing_server_config_validator import ProcessingServerConfigValidator +from .resource_list_validator import OcrdResourceListValidator +from .workspace_validator import WorkspaceValidator from .xsd_mets_validator import XsdMetsValidator from .xsd_page_validator import XsdPageValidator -from .processing_server_config_validator import ProcessingServerConfigValidator -from .ocrd_network_message_validator import OcrdNetworkMessageValidator +from .xsd_validator import XsdValidator diff --git a/src/ocrd_validators/constants.py b/src/ocrd_validators/constants.py index fc1ff445ae..1e1786bcbc 100644 --- a/src/ocrd_validators/constants.py +++ b/src/ocrd_validators/constants.py @@ -2,7 +2,7 @@ Constants for ocrd_validators. """ import yaml -from pkg_resources import resource_string, resource_filename +from pkg_resources import resource_filename, resource_string __all__ = [ 'PROCESSING_SERVER_CONFIG_SCHEMA', diff --git a/src/ocrd_validators/json_validator.py b/src/ocrd_validators/json_validator.py index c920fc7c2d..a2adae6126 100644 --- a/src/ocrd_validators/json_validator.py +++ b/src/ocrd_validators/json_validator.py @@ -3,10 +3,12 @@ """ import json -from jsonschema import Draft6Validator, validators # pylint: disable=import-error +from jsonschema import (Draft6Validator, # pylint: disable=import-error + validators) from ocrd_models import ValidationReport + # http://python-jsonschema.readthedocs.io/en/latest/faq/ def extend_with_default(validator_class): """ diff --git a/src/ocrd_validators/ocrd_network_message_validator.py b/src/ocrd_validators/ocrd_network_message_validator.py index 486efea433..efba2262c7 100644 --- a/src/ocrd_validators/ocrd_network_message_validator.py +++ b/src/ocrd_validators/ocrd_network_message_validator.py @@ -1,10 +1,7 @@ """ Validating ocrd-network messages """ -from .constants import ( - MESSAGE_SCHEMA_PROCESSING, - MESSAGE_SCHEMA_RESULT -) +from .constants import MESSAGE_SCHEMA_PROCESSING, MESSAGE_SCHEMA_RESULT from .json_validator import JsonValidator diff --git a/src/ocrd_validators/ocrd_zip_validator.py b/src/ocrd_validators/ocrd_zip_validator.py index 5f9d21f8c7..e174c68559 100644 --- a/src/ocrd_validators/ocrd_zip_validator.py +++ b/src/ocrd_validators/ocrd_zip_validator.py @@ -3,16 +3,18 @@ See `spec `_. """ -from tempfile import mkdtemp from shutil import rmtree +from tempfile import mkdtemp -from ocrd_utils import getLogger, unzip_file_to_dir - -from bagit import Bag, BagValidationError # pylint: disable=no-name-in-module -from bagit_profile import Profile, ProfileValidationError # pylint: disable=no-name-in-module +from bagit import Bag, BagValidationError # pylint: disable=no-name-in-module +from bagit_profile import (Profile, # pylint: disable=no-name-in-module + ProfileValidationError) -from .constants import OCRD_BAGIT_PROFILE, OCRD_BAGIT_PROFILE_URL, TMP_BAGIT_PREFIX from ocrd_models import ValidationReport +from ocrd_utils import getLogger, unzip_file_to_dir + +from .constants import (OCRD_BAGIT_PROFILE, OCRD_BAGIT_PROFILE_URL, + TMP_BAGIT_PREFIX) # # ------------------------------------------------- diff --git a/src/ocrd_validators/page_validator.py b/src/ocrd_validators/page_validator.py index 41ce0b9f94..3be63c3a5e 100644 --- a/src/ocrd_validators/page_validator.py +++ b/src/ocrd_validators/page_validator.py @@ -2,35 +2,24 @@ API for validating `OcrdPage <../ocrd_models/ocrd_models.ocrd_page.html>`_. """ import re -from shapely.geometry import Polygon, LineString + +from shapely.geometry import LineString, Polygon from shapely.validation import explain_validity -from ocrd_utils import getLogger, polygon_from_points, deprecated_alias -from ocrd_models.ocrd_page import parse from ocrd_modelfactory import page_from_file - -from ocrd_models.ocrd_page import ( - PcGtsType, - PageType, - TextRegionType, - TextLineType, - WordType, - GlyphType, - TextEquivType -) -from ocrd_models.ocrd_page_generateds import ( - RegionType, - ReadingDirectionSimpleType, - TextLineOrderSimpleType, - RegionRefType, - RegionRefIndexedType, - OrderedGroupType, - OrderedGroupIndexedType, - UnorderedGroupType, - UnorderedGroupIndexedType, -) from ocrd_models import ValidationReport - +from ocrd_models.ocrd_page import (GlyphType, PageType, PcGtsType, + TextEquivType, TextLineType, TextRegionType, + WordType, parse) +from ocrd_models.ocrd_page_generateds import (OrderedGroupIndexedType, + OrderedGroupType, + ReadingDirectionSimpleType, + RegionRefIndexedType, + RegionRefType, RegionType, + TextLineOrderSimpleType, + UnorderedGroupIndexedType, + UnorderedGroupType) +from ocrd_utils import deprecated_alias, getLogger, polygon_from_points _HIERARCHY = [ # page can contain different types of regions diff --git a/src/ocrd_validators/parameter_validator.py b/src/ocrd_validators/parameter_validator.py index 20dd6ff2b7..3ddde19434 100644 --- a/src/ocrd_validators/parameter_validator.py +++ b/src/ocrd_validators/parameter_validator.py @@ -1,7 +1,7 @@ """ Validate parameters against ocrd-tool.json. """ -from .json_validator import JsonValidator, DefaultValidatingDraft6Validator +from .json_validator import DefaultValidatingDraft6Validator, JsonValidator # # ------------------------------------------------- diff --git a/src/ocrd_validators/resource_list_validator.py b/src/ocrd_validators/resource_list_validator.py index 72a11c34de..3e9b273be1 100644 --- a/src/ocrd_validators/resource_list_validator.py +++ b/src/ocrd_validators/resource_list_validator.py @@ -4,7 +4,7 @@ See `specs `_. """ from .constants import RESOURCE_LIST_SCHEMA -from .json_validator import JsonValidator, DefaultValidatingDraft6Validator +from .json_validator import DefaultValidatingDraft6Validator, JsonValidator # # ------------------------------------------------- diff --git a/src/ocrd_validators/workspace_validator.py b/src/ocrd_validators/workspace_validator.py index 60014e2218..5bdc110292 100644 --- a/src/ocrd_validators/workspace_validator.py +++ b/src/ocrd_validators/workspace_validator.py @@ -2,17 +2,18 @@ Validating a workspace. """ import re -from traceback import format_exc from pathlib import Path +from traceback import format_exc -from ocrd_utils import getLogger, MIMETYPE_PAGE, pushd_popd, is_local_filename, DEFAULT_METS_BASENAME -from ocrd_models import ValidationReport from ocrd_modelfactory import page_from_file +from ocrd_models import ValidationReport +from ocrd_utils import (DEFAULT_METS_BASENAME, MIMETYPE_PAGE, getLogger, + is_local_filename, pushd_popd) from .constants import FILE_GROUP_CATEGORIES, FILE_GROUP_PREFIX from .page_validator import PageValidator -from .xsd_page_validator import XsdPageValidator from .xsd_mets_validator import XsdMetsValidator +from .xsd_page_validator import XsdPageValidator # # ------------------------------------------------- diff --git a/src/ocrd_validators/xsd_mets_validator.py b/src/ocrd_validators/xsd_mets_validator.py index 87e524e17c..0cbb02d7aa 100644 --- a/src/ocrd_validators/xsd_mets_validator.py +++ b/src/ocrd_validators/xsd_mets_validator.py @@ -1,5 +1,6 @@ -from .xsd_validator import XsdValidator from .constants import XSD_METS_URL +from .xsd_validator import XsdValidator + class XsdMetsValidator(XsdValidator): """ diff --git a/src/ocrd_validators/xsd_page_validator.py b/src/ocrd_validators/xsd_page_validator.py index eec6105f6d..c9de1854e3 100644 --- a/src/ocrd_validators/xsd_page_validator.py +++ b/src/ocrd_validators/xsd_page_validator.py @@ -1,5 +1,6 @@ -from .xsd_validator import XsdValidator from .constants import XSD_PAGE_URL +from .xsd_validator import XsdValidator + class XsdPageValidator(XsdValidator): """ diff --git a/src/ocrd_validators/xsd_validator.py b/src/ocrd_validators/xsd_validator.py index 81b9457564..17aa44b93e 100644 --- a/src/ocrd_validators/xsd_validator.py +++ b/src/ocrd_validators/xsd_validator.py @@ -3,6 +3,7 @@ """ from pathlib import Path + from lxml import etree as ET from ocrd_models import ValidationReport diff --git a/tests/base.py b/tests/base.py index 53f393e08d..952eb0a325 100644 --- a/tests/base.py +++ b/tests/base.py @@ -1,17 +1,20 @@ # pylint: disable=unused-import -from os.path import dirname, realpath -from os import chdir -from contextlib import contextmanager -import sys -import logging -import io import collections -from unittest import TestCase as VanillaTestCase, skip, main as unittests_main +import io +import logging +import sys +from contextlib import contextmanager +from os import chdir +from os.path import dirname, realpath +from unittest import TestCase as VanillaTestCase +from unittest import main as unittests_main +from unittest import skip + import pytest -from ocrd_utils import disableLogging, initLogging, getLogger -from ocrd_models import OcrdMets +from ocrd_models import OcrdMets +from ocrd_utils import disableLogging, getLogger, initLogging from tests.assets import assets, copy_of_directory diff --git a/tests/cli/test_bashlib.py b/tests/cli/test_bashlib.py index 8a6e40b029..a18e2c65d4 100644 --- a/tests/cli/test_bashlib.py +++ b/tests/cli/test_bashlib.py @@ -1,23 +1,24 @@ -from tests.base import CapturingTestCase as TestCase, main, assets, copy_of_directory - -from pkg_resources import parse_version -import os, sys -import traceback +import json +import os +import pathlib import subprocess +import sys import tempfile -import pathlib -import yaml -import json +import traceback from pathlib import Path -from ocrd.cli.bashlib import bashlib_cli +import yaml +from pkg_resources import parse_version +from ocrd.cli.bashlib import bashlib_cli from ocrd.constants import BASHLIB_FILENAME -from ocrd_utils.constants import VERSION, MIMETYPE_PAGE -from ocrd_validators.constants import BAGIT_TXT from ocrd_models.constants import TAG_MODS_IDENTIFIER - from ocrd_utils import pushd_popd +from ocrd_utils.constants import MIMETYPE_PAGE, VERSION +from ocrd_validators.constants import BAGIT_TXT +from tests.base import CapturingTestCase as TestCase +from tests.base import assets, copy_of_directory, main + class TestBashlibCli(TestCase): diff --git a/tests/cli/test_log.py b/tests/cli/test_log.py index c63d78c318..cb92e6a9de 100644 --- a/tests/cli/test_log.py +++ b/tests/cli/test_log.py @@ -1,13 +1,15 @@ -import click -from ocrd.cli import log_cli +import logging as python_logging from os import environ as ENV -# pylint: disable=import-error, no-name-in-module -from tests.base import CapturingTestCase as TestCase, main, assets, copy_of_directory +import click +from ocrd.cli import log_cli from ocrd.decorators import ocrd_loglevel -from ocrd_utils import setOverrideLogLevel, logging, disableLogging -import logging as python_logging +from ocrd_utils import disableLogging, logging, setOverrideLogLevel +# pylint: disable=import-error, no-name-in-module +from tests.base import CapturingTestCase as TestCase +from tests.base import assets, copy_of_directory, main + @click.group() @ocrd_loglevel diff --git a/tests/cli/test_process.py b/tests/cli/test_process.py index d0e5dc2129..3b50a40344 100644 --- a/tests/cli/test_process.py +++ b/tests/cli/test_process.py @@ -1,7 +1,8 @@ from ocrd.cli import process_cli -from ocrd_utils import pushd_popd, disableLogging +from ocrd_utils import disableLogging, pushd_popd +from tests.base import CapturingTestCase as TestCase +from tests.base import assets, copy_of_directory, main -from tests.base import CapturingTestCase as TestCase, main, assets, copy_of_directory class TestLogCli(TestCase): diff --git a/tests/cli/test_resmgr.py b/tests/cli/test_resmgr.py index 6cec6225b8..029950f944 100644 --- a/tests/cli/test_resmgr.py +++ b/tests/cli/test_resmgr.py @@ -1,11 +1,12 @@ from pathlib import Path +from tempfile import TemporaryDirectory + from click.testing import CliRunner from pytest import fixture -from tempfile import TemporaryDirectory -from ocrd_utils import disableLogging, directory_size from ocrd.cli.resmgr import resmgr_cli from ocrd.resource_manager import OcrdResourceManager +from ocrd_utils import directory_size, disableLogging runner = CliRunner() executable = 'ocrd-dummy' diff --git a/tests/cli/test_validate.py b/tests/cli/test_validate.py index 36ee3e5995..063c133c09 100644 --- a/tests/cli/test_validate.py +++ b/tests/cli/test_validate.py @@ -1,18 +1,16 @@ -from json import loads, dumps +from json import dumps, loads from pathlib import Path from tempfile import TemporaryDirectory from click.testing import CliRunner +from ocrd.cli.validate import validate_cli +from ocrd.resolver import Resolver +from ocrd_utils import pushd_popd # pylint: disable=import-error, no-name-in-module -from tests.base import main, assets +from tests.base import assets, main from tests.data.wf_testcase import TestCase -from ocrd_utils import pushd_popd -from ocrd.resolver import Resolver - -from ocrd.cli.validate import validate_cli - OCRD_TOOL = ''' { "git_url": "https://github.com/ocr-d/foo", diff --git a/tests/cli/test_workspace.py b/tests/cli/test_workspace.py index e96a391369..68c7b22d85 100644 --- a/tests/cli/test_workspace.py +++ b/tests/cli/test_workspace.py @@ -1,21 +1,23 @@ -from os.path import join, exists -from pathlib import Path +import sys +from contextlib import contextmanager from filecmp import dircmp +from io import StringIO +from os.path import exists, join +from pathlib import Path from shutil import copytree from tempfile import TemporaryDirectory -from io import StringIO -from contextlib import contextmanager -import sys -from click.testing import CliRunner import pytest +from click.testing import CliRunner +from ocrd import Resolver +from ocrd.cli.workspace import workspace_cli +from ocrd_utils import (disableLogging, initLogging, pushd_popd, + setOverrideLogLevel) # pylint: disable=import-error, no-name-in-module -from tests.base import CapturingTestCase as TestCase, assets, copy_of_directory, main +from tests.base import CapturingTestCase as TestCase +from tests.base import assets, copy_of_directory, main -from ocrd_utils import initLogging, pushd_popd, setOverrideLogLevel, disableLogging -from ocrd.cli.workspace import workspace_cli -from ocrd import Resolver @contextmanager def mock_stdin(inp): diff --git a/tests/data/__init__.py b/tests/data/__init__.py index 93a2ea49a9..5863140f59 100644 --- a/tests/data/__init__.py +++ b/tests/data/__init__.py @@ -1,5 +1,6 @@ import json import os + from ocrd import Processor from ocrd_utils import make_file_id diff --git a/tests/data/wf_testcase.py b/tests/data/wf_testcase.py index 2ea97ed212..02ed7e1e8a 100644 --- a/tests/data/wf_testcase.py +++ b/tests/data/wf_testcase.py @@ -1,9 +1,9 @@ -import os import json -from tempfile import mkdtemp +import os from os.path import join -from shutil import rmtree from pathlib import Path +from shutil import rmtree +from tempfile import mkdtemp from tests.base import CapturingTestCase as BaseTestCase diff --git a/tests/model/mets_bench_extreme.py b/tests/model/mets_bench_extreme.py index 63b30e31db..808c49c9a5 100644 --- a/tests/model/mets_bench_extreme.py +++ b/tests/model/mets_bench_extreme.py @@ -3,11 +3,11 @@ from contextlib import contextmanager from time import time -from pytest import main, fixture, mark +from pytest import fixture, main, mark from ocrd import Resolver -from ocrd_utils import MIME_TO_EXT, getLogger from ocrd_models import OcrdMets +from ocrd_utils import MIME_TO_EXT, getLogger logger = getLogger('ocrd.benchmark.mets') diff --git a/tests/model/mets_bench_extreme_additional.py b/tests/model/mets_bench_extreme_additional.py index e699454e2b..5a901d46e2 100644 --- a/tests/model/mets_bench_extreme_additional.py +++ b/tests/model/mets_bench_extreme_additional.py @@ -3,11 +3,11 @@ from contextlib import contextmanager from time import time -from pytest import main, fixture, mark +from pytest import fixture, main, mark from ocrd import Resolver -from ocrd_utils import MIME_TO_EXT, getLogger from ocrd_models import OcrdMets +from ocrd_utils import MIME_TO_EXT, getLogger logger = getLogger('ocrd.benchmark.mets') diff --git a/tests/model/test_agent.py b/tests/model/test_agent.py index 57c741655e..727d15e3fa 100644 --- a/tests/model/test_agent.py +++ b/tests/model/test_agent.py @@ -1,11 +1,7 @@ # -*- coding: utf-8 -*- -from ocrd_models import ( - OcrdAgent -) -from tests.base import ( - main -) +from ocrd_models import OcrdAgent +from tests.base import main def test_init_role_no_name(): diff --git a/tests/model/test_exif.py b/tests/model/test_exif.py index f6771fb8ee..3e57894a1d 100644 --- a/tests/model/test_exif.py +++ b/tests/model/test_exif.py @@ -2,16 +2,12 @@ import sys -from PIL import Image, __version__ as pil_version - import pytest - -from tests.base import ( - assets, - main -) +from PIL import Image +from PIL import __version__ as pil_version from ocrd_models import OcrdExif +from tests.base import assets, main @pytest.mark.parametrize("path,width,height,xResolution,yResolution,resolution,resolutionUnit,photometricInterpretation,compression", [ diff --git a/tests/model/test_ocrd_file.py b/tests/model/test_ocrd_file.py index 98b9fa4240..7fdd36539b 100644 --- a/tests/model/test_ocrd_file.py +++ b/tests/model/test_ocrd_file.py @@ -2,15 +2,8 @@ import pytest -from tests.base import ( - main, - create_ocrd_file, - create_ocrd_file_with_defaults -) -from ocrd_models import ( - OcrdMets, - OcrdFile, -) +from ocrd_models import OcrdFile, OcrdMets +from tests.base import create_ocrd_file, create_ocrd_file_with_defaults, main def test_ocrd_file_without_id(): diff --git a/tests/model/test_ocrd_mets.py b/tests/model/test_ocrd_mets.py index 64ea1eccfe..43752c0abe 100644 --- a/tests/model/test_ocrd_mets.py +++ b/tests/model/test_ocrd_mets.py @@ -1,33 +1,20 @@ # -*- coding: utf-8 -*- -from datetime import datetime - -from os.path import join -from os import environ -from contextlib import contextmanager import shutil +from contextlib import contextmanager +from datetime import datetime from logging import StreamHandler -import lxml - -from tests.base import ( - main, - capture_log, - assets, -) - -from ocrd_utils import ( - initLogging, - disableLogging, - getLogger, - VERSION, - MIMETYPE_PAGE -) -from ocrd_models import ( - OcrdMets -) +from os import environ +from os.path import join +import lxml import pytest +from ocrd_models import OcrdMets +from ocrd_utils import (MIMETYPE_PAGE, VERSION, disableLogging, getLogger, + initLogging) +from tests.base import assets, capture_log, main + CACHING_ENABLED = [False, True] diff --git a/tests/model/test_ocrd_mets_bench.py b/tests/model/test_ocrd_mets_bench.py index ace6387336..d25f02e686 100644 --- a/tests/model/test_ocrd_mets_bench.py +++ b/tests/model/test_ocrd_mets_bench.py @@ -1,16 +1,14 @@ # -*- coding: utf-8 -*- +import pprint from contextlib import contextmanager from time import time -from pytest import main, fixture, mark +from pytest import fixture, main, mark from ocrd import Resolver -from ocrd_utils import MIME_TO_EXT, getLogger from ocrd_models import OcrdMets - -import pprint - +from ocrd_utils import MIME_TO_EXT, getLogger # LOG = getLogger('ocrd.benchmark.mets') diff --git a/tests/model/test_ocrd_page.py b/tests/model/test_ocrd_page.py index 82964e349c..c2122c9f77 100644 --- a/tests/model/test_ocrd_page.py +++ b/tests/model/test_ocrd_page.py @@ -2,27 +2,15 @@ import pytest -from tests.base import main, assets, create_ocrd_file_with_defaults - from ocrd_modelfactory import page_from_image +from ocrd_models.ocrd_page import (AlternativeImageType, GlyphType, + OrderedGroupIndexedType, PageType, + PcGtsType, ReadingOrderType, + RegionRefIndexedType, TextLineType, + TextRegionType, UnorderedGroupIndexedType, + WordType, parse, parseString, to_xml) from ocrd_models.ocrd_page_generateds import TextTypeSimpleType -from ocrd_models.ocrd_page import ( - AlternativeImageType, - PcGtsType, - PageType, - TextRegionType, - TextLineType, - OrderedGroupIndexedType, - UnorderedGroupIndexedType, - ReadingOrderType, - RegionRefIndexedType, - WordType, - GlyphType, - - parseString, - parse, - to_xml -) +from tests.base import assets, create_ocrd_file_with_defaults, main simple_page = """\ diff --git a/tests/processor/test_ocrd_dummy.py b/tests/processor/test_ocrd_dummy.py index 1f4ecc0552..2da90c432c 100644 --- a/tests/processor/test_ocrd_dummy.py +++ b/tests/processor/test_ocrd_dummy.py @@ -1,18 +1,20 @@ # pylint: disable=missing-module-docstring,missing-function-docstring,missing-class-docstring # pylint: disable=invalid-name,line-too-long -from io import BytesIO import os +from io import BytesIO from pathlib import Path from PIL import Image -from tests.base import TestCase, assets, main, copy_of_directory # pylint: disable=import-error, no-name-in-module from ocrd import Resolver, Workspace -from ocrd_utils import MIMETYPE_PAGE, pushd_popd -from ocrd_modelfactory import page_from_file from ocrd.processor.base import run_processor from ocrd.processor.builtin.dummy_processor import DummyProcessor +from ocrd_modelfactory import page_from_file +from ocrd_utils import MIMETYPE_PAGE, pushd_popd +from tests.base import ( # pylint: disable=import-error, no-name-in-module + TestCase, assets, copy_of_directory, main) + class TestDummyProcessor(TestCase): diff --git a/tests/processor/test_processor.py b/tests/processor/test_processor.py index c6d560ba1d..938ade6051 100644 --- a/tests/processor/test_processor.py +++ b/tests/processor/test_processor.py @@ -1,15 +1,20 @@ import json - -from tempfile import TemporaryDirectory from os.path import join -from tests.base import CapturingTestCase as TestCase, assets, main # pylint: disable=import-error, no-name-in-module -from tests.data import DummyProcessor, DummyProcessorWithRequiredParameters, DummyProcessorWithOutput, IncompleteProcessor +from tempfile import TemporaryDirectory -from ocrd_utils import MIMETYPE_PAGE, pushd_popd, initLogging, disableLogging +import pytest + +from ocrd.processor.base import Processor, run_cli, run_processor from ocrd.resolver import Resolver -from ocrd.processor.base import Processor, run_processor, run_cli +from ocrd_utils import MIMETYPE_PAGE, disableLogging, initLogging, pushd_popd +from tests.base import \ + CapturingTestCase as \ + TestCase # pylint: disable=import-error, no-name-in-module +from tests.base import assets, main +from tests.data import (DummyProcessor, DummyProcessorWithOutput, + DummyProcessorWithRequiredParameters, + IncompleteProcessor) -import pytest class TestProcessor(TestCase): diff --git a/tests/test_decorators.py b/tests/test_decorators.py index 992e173dc1..f0112a0179 100644 --- a/tests/test_decorators.py +++ b/tests/test_decorators.py @@ -1,20 +1,22 @@ import json -import click from contextlib import contextmanager -from click.testing import CliRunner +from os.path import exists, join from tempfile import TemporaryDirectory -from os.path import join, exists -from tests.base import CapturingTestCase as TestCase, assets, main, copy_of_directory # pylint: disable=import-error, no-name-in-module -from tests.data import DummyProcessor +import click +from click.testing import CliRunner from ocrd import Processor, Resolver -from ocrd.decorators import ( - ocrd_cli_options, - ocrd_loglevel, - ocrd_cli_wrap_processor, -) # pylint: disable=protected-access -from ocrd_utils import pushd_popd, VERSION as OCRD_VERSION, disableLogging, initLogging +from ocrd.decorators import ( # pylint: disable=protected-access + ocrd_cli_options, ocrd_cli_wrap_processor, ocrd_loglevel) +from ocrd_utils import VERSION as OCRD_VERSION +from ocrd_utils import disableLogging, initLogging, pushd_popd +from tests.base import \ + CapturingTestCase as \ + TestCase # pylint: disable=import-error, no-name-in-module +from tests.base import assets, copy_of_directory, main +from tests.data import DummyProcessor + @click.command() @ocrd_cli_options diff --git a/tests/test_logging.py b/tests/test_logging.py index 2e4e0861b5..600466ab26 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -1,22 +1,17 @@ -import pytest import logging from re import match from tempfile import TemporaryDirectory -from tests.base import CapturingTestCase as TestCase, main, FIFOIO, assets -from tests.data import DummyProcessor -from ocrd import Resolver, run_processor +import pytest -from ocrd_utils import ( - pushd_popd, - getLevelName, - setOverrideLogLevel, - disableLogging, - initLogging, - getLogger, - LOG_FORMAT, - LOG_TIMEFMT -) +from ocrd import Resolver, run_processor +from ocrd_utils import (LOG_FORMAT, LOG_TIMEFMT, disableLogging, getLevelName, + getLogger, initLogging, pushd_popd, + setOverrideLogLevel) +from tests.base import FIFOIO +from tests.base import CapturingTestCase as TestCase +from tests.base import assets, main +from tests.data import DummyProcessor # "00:00:00.000 " TIMEFMT_RE = r'\d\d:\d\d:\d\d\.(\d+)? ' diff --git a/tests/test_logging_conf.py b/tests/test_logging_conf.py index 898722a413..140d2eeff3 100644 --- a/tests/test_logging_conf.py +++ b/tests/test_logging_conf.py @@ -7,15 +7,10 @@ import shutil import sys -from ocrd_utils import pushd_popd -from ocrd_utils.logging import ( - initLogging, - getLogger, - disableLogging, -) - import pytest +from ocrd_utils import pushd_popd +from ocrd_utils.logging import disableLogging, getLogger, initLogging from tests.base import main # sys.path.append(os.path.dirname(os.path.realpath(__file__)) + '/../ocrd') diff --git a/tests/test_mets_server.py b/tests/test_mets_server.py index 59ee50e086..43ac46a81b 100644 --- a/tests/test_mets_server.py +++ b/tests/test_mets_server.py @@ -1,22 +1,25 @@ +from itertools import repeat +from multiprocessing import Pipe, Pool, Process, set_start_method + from pytest import fixture, raises + from tests.base import assets -from itertools import repeat -from multiprocessing import Process, Pool, Pipe, set_start_method # necessary for macos set_start_method("fork") -from shutil import rmtree, copytree -from os import remove, stat as os_stat +import stat +from os import remove +from os import stat as os_stat from os.path import exists -from time import sleep from pathlib import Path -import stat +from shutil import copytree, rmtree +from time import sleep from uuid import uuid4 from requests.exceptions import ConnectionError -from ocrd import Resolver, OcrdMetsServer, Workspace -from ocrd_utils import pushd_popd, MIMETYPE_PAGE +from ocrd import OcrdMetsServer, Resolver, Workspace +from ocrd_utils import MIMETYPE_PAGE, pushd_popd WORKSPACE_DIR = '/tmp/ocrd-mets-server' TRANSPORTS = ['/tmp/ocrd-mets-server.sock', 'http://127.0.0.1:12345'] diff --git a/tests/test_model_factory.py b/tests/test_model_factory.py index f7da141f19..080a6c6767 100644 --- a/tests/test_model_factory.py +++ b/tests/test_model_factory.py @@ -1,12 +1,9 @@ -from tests.base import TestCase, main, assets, create_ocrd_file, create_ocrd_file_with_defaults - -from ocrd_utils import MIMETYPE_PAGE +from ocrd_modelfactory import (exif_from_filename, page_from_file, + page_from_image) from ocrd_models import OcrdMets -from ocrd_modelfactory import ( - exif_from_filename, - page_from_image, - page_from_file -) +from ocrd_utils import MIMETYPE_PAGE +from tests.base import (TestCase, assets, create_ocrd_file, + create_ocrd_file_with_defaults, main) SAMPLE_IMG = assets.path_to('kant_aufklaerung_1784/data/OCR-D-IMG/INPUT_0017.tif') SAMPLE_PAGE = assets.path_to('kant_aufklaerung_1784/data/OCR-D-GT-PAGE/PAGE_0017_PAGE.xml') diff --git a/tests/test_resolver.py b/tests/test_resolver.py index bf9f4bb723..9f17ea1b4f 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -2,31 +2,17 @@ import os import shutil - -from pathlib import ( - Path -) -from requests import Session -from unittest.mock import ( - patch, - Mock -) -from PIL import ( - Image -) - -from ocrd_models.ocrd_page import OcrdPage +from pathlib import Path +from unittest.mock import Mock, patch import pytest - -from tests.base import ( - assets, - main -) +from PIL import Image +from requests import Session from ocrd.resolver import Resolver +from ocrd_models.ocrd_page import OcrdPage from ocrd_utils import pushd_popd - +from tests.base import assets, main # set pylint once on module level # pylint: disable=protected-access diff --git a/tests/test_resolver_oai.py b/tests/test_resolver_oai.py index 13025dd22b..3183f5b067 100644 --- a/tests/test_resolver_oai.py +++ b/tests/test_resolver_oai.py @@ -1,15 +1,15 @@ -from unittest.mock import patch -from pytest import fixture +from logging import Formatter, StreamHandler +from os.path import dirname, join from shutil import copy -from logging import StreamHandler, Formatter -from os.path import join, dirname +from unittest.mock import patch +from pytest import fixture from requests import Session -from tests.base import main, FIFOIO from ocrd.resolver import Resolver from ocrd_models.utils import extract_mets_from_oai_content -from ocrd_utils import getLogger, initLogging, LOG_FORMAT +from ocrd_utils import LOG_FORMAT, getLogger, initLogging +from tests.base import FIFOIO, main @fixture(name="response_dir") diff --git a/tests/test_resource_manager.py b/tests/test_resource_manager.py index fb6db3adc4..2bb5785310 100644 --- a/tests/test_resource_manager.py +++ b/tests/test_resource_manager.py @@ -1,14 +1,17 @@ import os from pathlib import Path -# import pdb + +from pytest import fixture, raises from ocrd.resource_manager import OcrdResourceManager from ocrd_utils import config from ocrd_utils.os import get_ocrd_tool_json - -from pytest import raises, fixture from tests.base import main +# import pdb + + + CONST_RESOURCE_YML = 'resources.yml' CONST_RESOURCE_URL_LAYOUT = 'https://github.com/tesseract-ocr/tessdata_best/raw/main/bos.traineddata' diff --git a/tests/test_task_sequence.py b/tests/test_task_sequence.py index 3ebe6db2b6..091b696267 100644 --- a/tests/test_task_sequence.py +++ b/tests/test_task_sequence.py @@ -1,18 +1,14 @@ import json -from tempfile import TemporaryDirectory from pathlib import Path +from tempfile import TemporaryDirectory -from tests.base import main, assets, copy_of_directory -from tests.data.wf_testcase import ( - TestCase, - - SAMPLE_NAME_REQUIRED_PARAM, - PARAM_JSON, -) - -from ocrd_utils import pushd_popd, MIMETYPE_PAGE, get_ocrd_tool_json from ocrd.resolver import Resolver -from ocrd.task_sequence import run_tasks, validate_tasks, ProcessorTask +from ocrd.task_sequence import ProcessorTask, run_tasks, validate_tasks +from ocrd_utils import MIMETYPE_PAGE, get_ocrd_tool_json, pushd_popd +from tests.base import assets, copy_of_directory, main +from tests.data.wf_testcase import (PARAM_JSON, SAMPLE_NAME_REQUIRED_PARAM, + TestCase) + class TestOcrdWfStep(TestCase): diff --git a/tests/test_utils.py b/tests/test_utils.py index d2093c465d..a73913d4ae 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,53 +1,25 @@ from os import getcwd -from tempfile import TemporaryDirectory, gettempdir from pathlib import Path +from tempfile import TemporaryDirectory, gettempdir from PIL import Image - -from tests.base import TestCase, main, assets, create_ocrd_file from pytest import raises, warns -from ocrd_utils import ( - abspath, - - assert_file_grp_cardinality, - make_file_id, - - bbox_from_points, - bbox_from_xywh, - - concat_padded, - is_local_filename, - get_local_filename, - is_string, - membername, - generate_range, - - nth_url_segment, - remove_non_path_from_url, - safe_filename, - parse_json_string_or_file, - set_json_key_value_overrides, - - partition_list, - - points_from_bbox, - points_from_x0y0x1y1, - points_from_xywh, - points_from_polygon, - - polygon_from_points, - polygon_from_x0y0x1y1, - - xywh_from_points, - xywh_from_polygon, - pushd_popd, - - MIME_TO_EXT, EXT_TO_MIME, - MIME_TO_PIL, PIL_TO_MIME, -) -from ocrd_models.utils import xmllint_format from ocrd_models import OcrdMets +from ocrd_models.utils import xmllint_format +from ocrd_utils import (EXT_TO_MIME, MIME_TO_EXT, MIME_TO_PIL, PIL_TO_MIME, + abspath, assert_file_grp_cardinality, bbox_from_points, + bbox_from_xywh, concat_padded, generate_range, + get_local_filename, is_local_filename, is_string, + make_file_id, membername, nth_url_segment, + parse_json_string_or_file, partition_list, + points_from_bbox, points_from_polygon, + points_from_x0y0x1y1, points_from_xywh, + polygon_from_points, polygon_from_x0y0x1y1, pushd_popd, + remove_non_path_from_url, safe_filename, + set_json_key_value_overrides, xywh_from_points, + xywh_from_polygon) +from tests.base import TestCase, assets, create_ocrd_file, main def test_abspath(): diff --git a/tests/test_version.py b/tests/test_version.py index adfb7ca989..5b66d69ec2 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -1,5 +1,6 @@ -from tests.base import TestCase, main from ocrd_utils import VERSION +from tests.base import TestCase, main + class TestVersion(TestCase): diff --git a/tests/test_workspace.py b/tests/test_workspace.py index 18dd233319..615860e54b 100644 --- a/tests/test_workspace.py +++ b/tests/test_workspace.py @@ -1,37 +1,31 @@ # -*- coding: utf-8 -*- -from os import chdir, curdir, walk, stat, chmod, umask -import shutil import logging -from stat import filemode -from os.path import join, exists, abspath, basename, dirname -from shutil import copyfile, copytree as copytree_, rmtree -from pathlib import Path +import shutil from gzip import open as gzip_open +from os import chdir, chmod, curdir, stat, umask, walk +from os.path import abspath, basename, dirname, exists, join +from pathlib import Path +from shutil import copyfile +from shutil import copytree as copytree_ +from shutil import rmtree +from stat import filemode -from PIL import Image import numpy as np - import pytest +from PIL import Image -from tests.base import ( - assets, - main, - FIFOIO -) - -from ocrd_models import ( - OcrdFile, - OcrdMets -) -from ocrd_models.ocrd_page import parseString -from ocrd_models.ocrd_page import TextRegionType, CoordsType, AlternativeImageType -from ocrd_utils import polygon_mask, xywh_from_polygon, bbox_from_polygon, points_from_polygon -from ocrd_modelfactory import page_from_file from ocrd.resolver import Resolver from ocrd.workspace import Workspace from ocrd.workspace_backup import WorkspaceBackupManager +from ocrd_modelfactory import page_from_file +from ocrd_models import OcrdFile, OcrdMets +from ocrd_models.ocrd_page import (AlternativeImageType, CoordsType, + TextRegionType, parseString) +from ocrd_utils import (bbox_from_polygon, points_from_polygon, polygon_mask, + xywh_from_polygon) from ocrd_validators import WorkspaceValidator +from tests.base import FIFOIO, assets, main TMP_FOLDER = '/tmp/test-core-workspace' SRC_METS = assets.path_to('kant_aufklaerung_1784/data/mets.xml') diff --git a/tests/test_workspace_remove.py b/tests/test_workspace_remove.py index feb7937ca9..11b1703c40 100644 --- a/tests/test_workspace_remove.py +++ b/tests/test_workspace_remove.py @@ -4,17 +4,11 @@ import shutil import lxml.etree as ET -from tests.base import main - import pytest -from ocrd.resolver import ( - Resolver -) - -from ocrd.workspace import ( - Workspace -) +from ocrd.resolver import Resolver +from ocrd.workspace import Workspace +from tests.base import main def test_workspace_init_missing_mets(): diff --git a/tests/utils/test_config.py b/tests/utils/test_config.py index 99595a864c..d36d12747d 100644 --- a/tests/utils/test_config.py +++ b/tests/utils/test_config.py @@ -1,9 +1,10 @@ -from os import environ from contextlib import contextmanager +from os import environ from pytest import raises -from ocrd_utils.config import config, OcrdEnvConfig +from ocrd_utils.config import OcrdEnvConfig, config + @contextmanager def temp_env_var(k, v): diff --git a/tests/utils/test_deprecate.py b/tests/utils/test_deprecate.py index c201d2d48c..5b7c12cf66 100644 --- a/tests/utils/test_deprecate.py +++ b/tests/utils/test_deprecate.py @@ -1,8 +1,9 @@ import click from ocrd.cli import command_with_replaced_help +from tests.base import CapturingTestCase as TestCase +from tests.base import main -from tests.base import CapturingTestCase as TestCase, main class TestDeprectateUtils(TestCase): diff --git a/tests/utils/test_image.py b/tests/utils/test_image.py index 936add2728..835c503019 100644 --- a/tests/utils/test_image.py +++ b/tests/utils/test_image.py @@ -1,7 +1,9 @@ -from pytest import main from PIL import Image +from pytest import main + from ocrd_utils.image import rotate_image + def test_32bit_fill(): img = Image.new('F', (200, 100), 1) rotate_image(img, 0.1, fill='background', transparency=False) diff --git a/tests/utils/test_os.py b/tests/utils/test_os.py index a651ef608f..8d54ac5907 100644 --- a/tests/utils/test_os.py +++ b/tests/utils/test_os.py @@ -1,16 +1,15 @@ -from tempfile import mkdtemp -from tests.base import TestCase, main, assets -from shutil import rmtree -from pathlib import Path -from os import environ as ENV, getcwd -from os.path import expanduser, join import sys +from os import environ as ENV +from os import getcwd +from os.path import expanduser, join +from pathlib import Path +from shutil import rmtree +from tempfile import mkdtemp + +from ocrd_utils.os import (guess_media_type, list_resource_candidates, + redirect_stderr_and_stdout_to_file) +from tests.base import TestCase, assets, main -from ocrd_utils.os import ( - list_resource_candidates, - redirect_stderr_and_stdout_to_file, - guess_media_type, -) class TestOsUtils(TestCase): diff --git a/tests/validator/test_json_validator.py b/tests/validator/test_json_validator.py index 8a8387d4b6..c9aa8bb873 100644 --- a/tests/validator/test_json_validator.py +++ b/tests/validator/test_json_validator.py @@ -1,5 +1,7 @@ +from ocrd_validators.json_validator import (DefaultValidatingDraft6Validator, + JsonValidator) from tests.base import TestCase, main -from ocrd_validators.json_validator import JsonValidator, DefaultValidatingDraft6Validator + class TestParameterValidator(TestCase): diff --git a/tests/validator/test_ocrd_tool_validator.py b/tests/validator/test_ocrd_tool_validator.py index 3ad40d8645..4f099d95f6 100644 --- a/tests/validator/test_ocrd_tool_validator.py +++ b/tests/validator/test_ocrd_tool_validator.py @@ -1,8 +1,8 @@ import json -from tests.base import TestCase, main # pylint: disable=import-error,no-name-in-module - from ocrd_validators import OcrdToolValidator +from tests.base import ( # pylint: disable=import-error,no-name-in-module + TestCase, main) skeleton = ''' { diff --git a/tests/validator/test_ocrd_zip_validator.py b/tests/validator/test_ocrd_zip_validator.py index 4c39c3ea33..3c84f8c29d 100644 --- a/tests/validator/test_ocrd_zip_validator.py +++ b/tests/validator/test_ocrd_zip_validator.py @@ -2,14 +2,15 @@ from shutil import copytree, rmtree from tempfile import mkdtemp -from tests.base import TestCase, main, assets # pylint: disable=import-error,no-name-in-module +from ocrd.resolver import Resolver +from ocrd.workspace import Workspace +from ocrd.workspace_bagger import WorkspaceBagger +from ocrd_validators import OcrdZipValidator +from tests.base import ( # pylint: disable=import-error,no-name-in-module + TestCase, assets, main) # from bagit import BagValidationError # pylint: disable=no-name-in-module -from ocrd_validators import OcrdZipValidator -from ocrd.workspace import Workspace -from ocrd.workspace_bagger import WorkspaceBagger -from ocrd.resolver import Resolver class TestOcrdZipValidator(TestCase): diff --git a/tests/validator/test_page_validator.py b/tests/validator/test_page_validator.py index 79e92d90fa..6de02ed143 100644 --- a/tests/validator/test_page_validator.py +++ b/tests/validator/test_page_validator.py @@ -1,9 +1,10 @@ -from tests.base import TestCase, assets, main # pylint: disable=import-error,no-name-in-module from ocrd.resolver import Resolver -from ocrd_validators import PageValidator -from ocrd_validators.page_validator import get_text, set_text, ConsistencyError -from ocrd_models.ocrd_page import parse, TextEquivType +from ocrd_models.ocrd_page import TextEquivType, parse from ocrd_utils import pushd_popd +from ocrd_validators import PageValidator +from ocrd_validators.page_validator import ConsistencyError, get_text, set_text +from tests.base import ( # pylint: disable=import-error,no-name-in-module + TestCase, assets, main) FAULTY_GLYPH_PAGE_FILENAME = assets.path_to('glyph-consistency/data/OCR-D-GT-PAGE/FAULTY_GLYPHS.xml') diff --git a/tests/validator/test_parameter_validator.py b/tests/validator/test_parameter_validator.py index f0d9d41d2c..b574cf4bcd 100644 --- a/tests/validator/test_parameter_validator.py +++ b/tests/validator/test_parameter_validator.py @@ -1,5 +1,6 @@ -from tests.base import TestCase, main from ocrd_validators import ParameterValidator +from tests.base import TestCase, main + class TestParameterValidator(TestCase): diff --git a/tests/validator/test_report.py b/tests/validator/test_report.py index 3eccb08f61..e8c4684d24 100644 --- a/tests/validator/test_report.py +++ b/tests/validator/test_report.py @@ -1,5 +1,7 @@ -from tests.base import TestCase, main # pylint: disable=import-error,no-name-in-module from ocrd_models import ValidationReport +from tests.base import ( # pylint: disable=import-error,no-name-in-module + TestCase, main) + class TestValidationReport(TestCase): diff --git a/tests/validator/test_resource_list_validator.py b/tests/validator/test_resource_list_validator.py index eb95d9b1ea..4ee22d3b47 100644 --- a/tests/validator/test_resource_list_validator.py +++ b/tests/validator/test_resource_list_validator.py @@ -1,9 +1,11 @@ import json -from tests.base import TestCase, main # pylint: disable=import-error,no-name-in-module from pytest import fixture from ocrd_validators import OcrdResourceListValidator +from tests.base import ( # pylint: disable=import-error,no-name-in-module + TestCase, main) + @fixture def reslist(): diff --git a/tests/validator/test_workspace_backup.py b/tests/validator/test_workspace_backup.py index 0d0bc141ca..efae8b3a53 100644 --- a/tests/validator/test_workspace_backup.py +++ b/tests/validator/test_workspace_backup.py @@ -2,11 +2,12 @@ from shutil import copytree, rmtree from tempfile import mkdtemp -from tests.base import TestCase, main, assets # pylint: disable=import-error,no-name-in-module - -from ocrd.workspace_backup import WorkspaceBackupManager from ocrd import Workspace from ocrd.resolver import Resolver +from ocrd.workspace_backup import WorkspaceBackupManager +from tests.base import ( # pylint: disable=import-error,no-name-in-module + TestCase, assets, main) + class TestWorkspaceBackup(TestCase): diff --git a/tests/validator/test_workspace_bagger.py b/tests/validator/test_workspace_bagger.py index 3cd0fb621e..37e000ded2 100644 --- a/tests/validator/test_workspace_bagger.py +++ b/tests/validator/test_workspace_bagger.py @@ -1,18 +1,18 @@ from os import makedirs -from os.path import join, abspath, exists -from shutil import copytree, rmtree, move, make_archive -from tempfile import mkdtemp +from os.path import abspath, exists, join from pathlib import Path -from bagit import _load_tag_file, Bag +from shutil import copytree, make_archive, move, rmtree +from tempfile import mkdtemp -from tests.base import TestCase, main, assets # pylint: disable=import-error,no-name-in-module +from bagit import Bag, _load_tag_file +from pytest import fixture, raises -from ocrd.workspace import Workspace -from ocrd.workspace_bagger import WorkspaceBagger, BACKUPDIR from ocrd.resolver import Resolver -from ocrd_utils import unzip_file_to_dir, pushd_popd, initLogging - -from pytest import fixture, raises +from ocrd.workspace import Workspace +from ocrd.workspace_bagger import BACKUPDIR, WorkspaceBagger +from ocrd_utils import initLogging, pushd_popd, unzip_file_to_dir +from tests.base import ( # pylint: disable=import-error,no-name-in-module + TestCase, assets, main) README_FILE = abspath('README.md') diff --git a/tests/validator/test_workspace_validator.py b/tests/validator/test_workspace_validator.py index 39c03bd842..94d6d611dc 100644 --- a/tests/validator/test_workspace_validator.py +++ b/tests/validator/test_workspace_validator.py @@ -1,16 +1,18 @@ import os -from tempfile import TemporaryDirectory -from pathlib import Path from os.path import join +from pathlib import Path from shutil import copytree +from tempfile import TemporaryDirectory + import pytest -from ocrd_utils import pushd_popd from ocrd.resolver import Resolver +from ocrd_utils import pushd_popd from ocrd_validators import WorkspaceValidator from ocrd_validators.page_validator import ConsistencyError +from tests.base import ( # pylint: disable=import-error,no-name-in-module + TestCase, assets, copy_of_directory, main) -from tests.base import TestCase, assets, main, copy_of_directory # pylint: disable=import-error,no-name-in-module class TestWorkspaceValidator(TestCase): diff --git a/tests/validator/test_xsd_validator.py b/tests/validator/test_xsd_validator.py index 4d4deadc15..7642260112 100644 --- a/tests/validator/test_xsd_validator.py +++ b/tests/validator/test_xsd_validator.py @@ -3,13 +3,14 @@ from pathlib import Path from tempfile import TemporaryDirectory -from tests.base import TestCase, main, assets from ocrd import Resolver from ocrd_models.constants import METS_XML_EMPTY -from ocrd_validators import XsdValidator, XsdMetsValidator, XsdPageValidator +from ocrd_validators import XsdMetsValidator, XsdPageValidator, XsdValidator from ocrd_validators.constants import XSD_METS_URL, XSD_PAGE_URL +from tests.base import TestCase, assets, main from tests.model.test_ocrd_page import simple_page + class TestXsdValidator(TestCase): def setUp(self):