diff --git a/Dockerfile b/Dockerfile index 53af2f2..a0b92aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,10 +5,12 @@ # cds-migrator-kit is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. -FROM python:3.6 +FROM python:3.9 RUN apt-get update -y && apt-get upgrade -y -RUN apt-get install -y git curl vim +RUN apt-get install -y git curl vim build-essential python3-dev \ +libldap2-dev libsasl2-dev slapd ldap-utils tox \ +lcov valgrind RUN pip install --upgrade setuptools wheel pip pipenv uwsgi uwsgitop uwsgi-tools RUN python -m site diff --git a/cds_migrator_kit/errors.py b/cds_migrator_kit/errors.py new file mode 100644 index 0000000..a8f1f28 --- /dev/null +++ b/cds_migrator_kit/errors.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2024 CERN. +# +# cds-migrator-kit is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Exceptions.""" + +from dojson.errors import DoJSONException + +################################################################# +# CDS-ILS Migrator Exceptions +################################################################# + +class LossyConversion(DoJSONException): + """Data lost during migration.""" + + def __init__(self, *args, **kwargs): + """Exception custom initialisation.""" + self.missing = kwargs.pop("missing", None) + self.message = self.description = "Lossy conversion: {0}".format( + self.missing or "" + ) + super().__init__(*args, **kwargs) + + +class RecordNotDeletable(DoJSONException): + """Record is not marked as deletable.""" + + def __init__(self, *args, **kwargs): + """Exception custom initialisation.""" + self.message = self.description = "Record is not marked as deletable" + super().__init__(*args, **kwargs) + + +class ProviderNotAllowedDeletion(DoJSONException): + """Provider is not allowed to delete records.""" + + def __init__(self, *args, **kwargs): + """Exception custom initialisation.""" + self.provider = kwargs.pop("provider", None) + self.message = self.description = ( + "This provider {0} is not allowed to delete records".format(self.provider) + ) + super().__init__(*args, **kwargs) + + +class CDSImporterException(DoJSONException): + """CDSDoJSONException class.""" + + def __init__(self, *args, **kwargs): + """Constructor.""" + self.subfield = kwargs.get("subfield", "") + message = kwargs.get("message", None) + if message: + self.message = message + + # because of ILSRestException class attributes + self.description = self.message + + super(CDSImporterException, self).__init__(*args) + + +class RecordModelMissing(CDSImporterException): + """Missing record model exception.""" + + message = "[Record did not match any available model]" + + +class UnexpectedValue(CDSImporterException): + """The corresponding value is unexpected.""" + + message = "[UNEXPECTED INPUT VALUE]" + + +class MissingRequiredField(CDSImporterException): + """The corresponding value is required.""" + + message = "[MISSING REQUIRED FIELD]" + + +class ManualImportRequired(CDSImporterException): + """The corresponding field should be manually migrated.""" + + message = "[MANUAL IMPORT REQUIRED]" + + +class DocumentImportError(CDSImporterException): + """Document import exception.""" + + message = "[DOCUMENT IMPORT ERROR]" + + +class SeriesImportError(CDSImporterException): + """Document import exception.""" + + message = "[SERIES IMPORT ERROR]" + + +class UnknownProvider(CDSImporterException): + """Unknown provider exception.""" + + message = "Unknown record provider." + + +class InvalidProvider(CDSImporterException): + """Invalid provider exception.""" + + message = "Invalid record provider." + + +class SimilarityMatchUnavailable(CDSImporterException): + """Similarity match unavailable exception.""" + + message = ( + "Title similarity matching cannot be performed for " + "this record. Please import it manually." + ) + + +############################################################################### +# Migration exceptions +############################################################################### + + +class DumpRevisionException(Exception): + """Exception for dump revision.""" + + +class JSONConversionException(Exception): + """JSON Conversion Exception in migration.""" + + +class MigrationException(Exception): + """Base exception for CDS-ILS migration errors.""" + + +class DocumentMigrationError(MigrationException): + """Raised for multipart migration errors.""" + + +class SeriesMigrationError(MigrationException): + """Raised for multipart migration errors.""" + + +class MultipartMigrationError(MigrationException): + """Raised for multipart migration errors.""" + + +class UserMigrationError(MigrationException): + """Raised for user migration errors.""" + + +class SerialMigrationError(MigrationException): + """Raised for serial migration errors.""" + + +class ItemMigrationError(MigrationException): + """Raised for item migration errors.""" + + +class LoanMigrationError(MigrationException): + """Raised for loan migration errors.""" + + +class EItemMigrationError(MigrationException): + """Raised for EItem migration errors.""" + + +class FileMigrationError(MigrationException): + """Raised for File migration errors.""" + + +class BorrowingRequestError(MigrationException): + """Raised for borrowing request migration errors.""" + + +class AcqOrderError(MigrationException): + """Raised for acquisition order migration errors.""" + + +class ProviderError(MigrationException): + """Raised for provider migration errors.""" + + +class RelationMigrationError(MigrationException): + """Raised for exceptions when migrating relations.""" diff --git a/cds_migrator_kit/ext.py b/cds_migrator_kit/ext.py index 2d04cd8..37ea85f 100644 --- a/cds_migrator_kit/ext.py +++ b/cds_migrator_kit/ext.py @@ -29,7 +29,7 @@ def init_app(self, app): self.init_config(app) set_logging() app.extensions['cds-migrator-kit'] = self - app.register_blueprint(blueprint) + app.register_blueprint(blueprint, name='cds_migrator_kit_records_bp') def init_config(self, app): """Initialize configuration.""" diff --git a/cds_migrator_kit/handlers.py b/cds_migrator_kit/handlers.py new file mode 100644 index 0000000..4ba8663 --- /dev/null +++ b/cds_migrator_kit/handlers.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2024 CERN. +# +# cds-migrator-kit is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""CDS Migrator Records logging handler.""" + +import logging + +cli_logger = logging.getLogger("migrator") +documents_logger = logging.getLogger("documents_logger") +items_logger = logging.getLogger("items_logger") + + +def migration_exception_handler(exc, output, key, value, rectype=None, **kwargs): + """Migration exception handling - log to files. + + :param exc: exception + :param output: generated output version + :param key: MARC field ID + :param value: MARC field value + :return: + """ + logger = logging.getLogger(f"{rectype}s_logger") + cli_logger.error( + "#RECID: #{0} - {1} MARC FIELD: *{2}*, input value: {3}, -> {4}, ".format( + output["legacy_recid"], exc.message, key, value, output + ) + ) + logger.error( + "MARC: {0}, INPUT VALUE: {1} ERROR: {2}" "".format(key, value, exc.message), + extra=dict(legacy_id=output["legacy_recid"], status="WARNING", new_pid=None), + ) \ No newline at end of file diff --git a/cds_migrator_kit/records/cli.py b/cds_migrator_kit/records/cli.py index e0010fb..1094fd6 100644 --- a/cds_migrator_kit/records/cli.py +++ b/cds_migrator_kit/records/cli.py @@ -12,14 +12,15 @@ import logging import click -from cds_ils.importer.providers.cds.models.document import model as book_model -from cds_ils.importer.providers.cds.models.journal import \ +from cds_dojson.marc21.models.books.book import model as book_model +from cds_dojson.marc21.models.books.journal import \ model as journal_model -from cds_ils.importer.providers.cds.models.multipart import \ +from cds_dojson.marc21.models.books.multipart import \ model as multipart_model -from cds_ils.importer.providers.cds.models.serial import model as serial_model -from cds_ils.importer.providers.cds.models.standard import \ +from cds_dojson.marc21.models.rdm.summer_student_report import model as summer_student_model +from cds_dojson.marc21.models.books.standard import \ model as standard_model +from cds_dojson.marc21.models.books.serial import model as serial_model from flask import current_app from flask.cli import with_appcontext @@ -124,7 +125,11 @@ def dryrun(sources, source_type, recid, rectype, model=None): params['dojson_model'] = book_model elif rectype == 'standard': params['dojson_model'] = standard_model + elif rectype == 'summer-student': + params['dojson_model'] = summer_student_model else: raise ValueError('invalid rectype: {}'.format(rectype)) + # import pdb; pdb.set_trace() + load_records(sources=sources, source_type=source_type, eager=True, rectype=rectype, **params) diff --git a/cds_migrator_kit/records/log.py b/cds_migrator_kit/records/log.py index e310db7..edfacc0 100644 --- a/cds_migrator_kit/records/log.py +++ b/cds_migrator_kit/records/log.py @@ -13,11 +13,9 @@ import logging import os -from cds_ils.importer.errors import ManualImportRequired, \ - MissingRequiredField, UnexpectedValue from flask import current_app -from fuzzywuzzy import fuzz +from cds_migrator_kit.errors import ManualImportRequired, MissingRequiredField, UnexpectedValue from cds_migrator_kit.records.errors import LossyConversion, \ RequiredFieldMissing from cds_migrator_kit.records.utils import clean_exception_message, \ @@ -67,6 +65,8 @@ def get_json_logger(cls, rectype): return DocumentJsonLogger() elif rectype == 'multipart': return MultipartJsonLogger() + elif rectype == 'summer-student': + return RdmJsonLogger() else: raise Exception('Invalid rectype: {}'.format(rectype)) @@ -181,6 +181,30 @@ def add_recid_to_stats(self, recid): def add_record(self, record): """Add record to collected records.""" self.records[record['legacy_recid']] = record + + +class RdmJsonLogger(JsonLogger): + """Log rdm record migration statistic to file controller.""" + + def __init__(self): + """Constructor.""" + super().__init__('rdm_stats.json', 'rdm_records.json') + + def add_recid_to_stats(self, recid): + """Add empty log item.""" + if recid not in self.stats: + self.stats[recid] = { + 'recid': recid, + 'manual_migration': [], + 'unexpected_value': [], + 'missing_required_field': [], + 'lost_data': [], + 'clean': True, + } + + def add_record(self, record): + """Add record to collected records.""" + self.records[record['legacy_recid']] = record class JournalJsonLogger(JsonLogger): diff --git a/cds_migrator_kit/records/records.py b/cds_migrator_kit/records/records.py index 57396f2..6683a99 100644 --- a/cds_migrator_kit/records/records.py +++ b/cds_migrator_kit/records/records.py @@ -14,17 +14,17 @@ import arrow from cds_dojson.marc21.utils import create_record -from cds_ils.importer import marc21 -from cds_ils.importer.errors import ManualImportRequired, \ - MissingRequiredField, UnexpectedValue -from cds_ils.migrator.xml_to_json_dump import CDSRecordDump +from cds_dojson.overdo import OverdoBase from flask import current_app +from cds_migrator_kit.errors import ManualImportRequired, MissingRequiredField, UnexpectedValue from cds_migrator_kit.records.errors import LossyConversion from cds_migrator_kit.records.handlers import migration_exception_handler +from cds_migrator_kit.xml_to_json_dump import CDSRecordDump cli_logger = logging.getLogger('migrator') +marc21 = OverdoBase(entry_point_models="cds.importer.models") class CDSMigKitDump(CDSRecordDump): """CDS record dump class.""" diff --git a/cds_migrator_kit/records/templates/cds_migrator_kit_records/base.html b/cds_migrator_kit/records/templates/cds_migrator_kit_records/base.html index a4e302c..c493e18 100644 --- a/cds_migrator_kit/records/templates/cds_migrator_kit_records/base.html +++ b/cds_migrator_kit/records/templates/cds_migrator_kit_records/base.html @@ -37,6 +37,9 @@
  • Multipart Monographs
    • +
  • + SSPN +
    • diff --git a/cds_migrator_kit/records/templates/cds_migrator_kit_records/summer-student.html b/cds_migrator_kit/records/templates/cds_migrator_kit_records/summer-student.html new file mode 100644 index 0000000..3c1a25d --- /dev/null +++ b/cds_migrator_kit/records/templates/cds_migrator_kit_records/summer-student.html @@ -0,0 +1,70 @@ +{# + Copyright (C) 2015-2018 CERN. + cds-migrator-kit is free software; you can redistribute it and/or modify it + under the terms of the MIT License; see LICENSE file for more details. +#} + +{%- extends config.CDS_MIGRATOR_KIT_BASE_TEMPLATE %} + +{%- block page_body %} + + + + + + + + + + + + + + {% for stat in stats_sorted_by_key %} + + + + + + + + + {% endfor %} + +
    RecidUnexpected ValueMissing requiredManual MigrationLost data fieldsDocument
    {{ stat.recid }} + {% for val in stat.unexpected_value %} + + {{ val.key }}{{ val.subfield or '' }}: {{ val.value }} +
    + {% endfor %} +     		+ {% for val in stat.missing_required_field %} + + {{ val.key }}{{ val.subfield or '' }}: {{ val.value }} +
    + {% endfor %} +     		+ {% for val in stat.manual_migration %} + + {{ val.key }}{{ val.subfield or '' }}: {{ val.value }} +
    + {% endfor %} +     		+ {% for val in stat.lost_data %} + {% for missing in val.missing %} + {{ missing }}
    + {% endfor %} + {% endfor %} +     		+ {% if not stat.lost_data %} + View + {% endif %} +
    + + + +{%- endblock %} diff --git a/cds_migrator_kit/xml_to_json_dump.py b/cds_migrator_kit/xml_to_json_dump.py new file mode 100644 index 0000000..8b409ba --- /dev/null +++ b/cds_migrator_kit/xml_to_json_dump.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2024 CERN. +# +# cds-migrator-kit is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""CDS Migrator Records MARCXML to JSON dump.""" +import logging + +import arrow +from cds_dojson.marc21.utils import create_record + +from cds_migrator_kit.errors import ( + ManualImportRequired, + MissingRequiredField, + UnexpectedValue, + JSONConversionException, + LossyConversion, +) +from cds_migrator_kit.handlers import migration_exception_handler +from cds_dojson.migrator import migrator_marc21 + +cli_logger = logging.getLogger("migrator") + + + +class CDSRecordDump(object): + """CDS record dump class.""" + + def __init__( + self, + data, + source_type="marcxml", + latest_only=True, + dojson_model=migrator_marc21, + ): + """Initialize.""" + self.data = data + self.source_type = source_type + self.latest_only = latest_only + self.dojson_model = dojson_model + self.revisions = None + self.files = None + + @property + def created(self): + """Get creation date.""" + return self.revisions[0][0] + + def prepare_revisions(self): + """Prepare revisions.""" + # get only the latest in any case + latest = self.data["record"][-1] + self.revisions = [self._prepare_revision(latest)] + + def prepare_files(self): + """Get files from data dump.""" + # Prepare files + files = {} + for f in self.data["files"]: + k = f["full_name"] + if k not in files: + files[k] = [] + files[k].append(f) + + # Sort versions + for k in files.keys(): + files[k].sort(key=lambda x: x["version"]) + + self.files = files + + def _prepare_revision(self, data): + timestamp = arrow.get(data["modification_datetime"]).datetime + + exception_handlers = { + UnexpectedValue: migration_exception_handler, + MissingRequiredField: migration_exception_handler, + ManualImportRequired: migration_exception_handler, + } + + if self.source_type == "marcxml": + marc_record = create_record(data["marcxml"]) + try: + json_converted_record = self.dojson_model.do( + marc_record, exception_handlers=exception_handlers + ) + except Exception as e: + raise JSONConversionException(e) + missing = self.dojson_model.missing(marc_record) + if missing: + raise LossyConversion(missing=missing) + return timestamp, json_converted_record + else: + return timestamp, data["json"] diff --git a/requirements.txt b/requirements.txt index e23e842..dca8360 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,150 +11,232 @@ # -e git+git://github.com/mitsuhiko/werkzeug.git#egg=Werkzeug # -e git+git://github.com/mitsuhiko/jinja2.git#egg=Jinja2 -gunicorn -# TODO: delete me when cds-dojson merge to master and released -pyOpenSSL --e git+https://github.com/CERNDocumentServer/cds-ils.git@master#egg=cds-ils -argparse==1.4.0 -alabaster==0.7.12 -alembic==1.4.2 -amqp==2.6.0 -apipkg==1.5 -appnope==0.1.0 -arrow==0.16.0 -atomicwrites==1.4.0 -attrs==19.3.0 -Babel==2.8.0 -backcall==0.2.0 +alabaster==0.7.16 +alembic==1.10.4 +amqp==5.2.0 +apipkg==3.0.2 +appdirs==1.4.4 +arrow==1.3.0 +asttokens==2.4.1 +async-timeout==4.0.3 +atomicwrites==1.4.1 +attrs==23.2.0 +Babel==2.15.0 base32-lib==1.0.2 -billiard==3.6.3.0 -bleach==3.1.5 -blinker==1.4 -celery==4.4.6 -certifi==2020.6.20 -cffi==1.14.0 -chardet==3.0.4 -check-manifest==0.42 -click==7.1.2 -click-default-group==1.2.2 +billiard==3.6.4.0 +bleach==6.1.0 +blinker==1.8.2 +build==1.2.1 +cachelib==0.9.0 +git+ssh://git@github.com/CERNDocumentServer/cds-dojson.git@f1b191323bbf60197a45a791779ab6ce5571df7e#egg=cds_dojson +celery==5.2.7 +certifi==2024.6.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +check-manifest==0.49 +click==8.1.7 +click-default-group==1.2.4 +click-didyoumean==0.3.1 +click-plugins==1.1.1 +click-repl==0.3.0 +counter-robots==2018.6 coverage==4.5.4 -cryptography==2.9.2 -decorator==4.4.2 -dnspython==1.16.0 -docutils==0.16 +cryptography==42.0.8 +decorator==5.1.1 +dictdiffer==0.9.0 +distlib==0.3.8 +dnspython==2.6.1 +Docker-Services-CLI==0.8.0 +docutils==0.21.2 dojson==1.4.0 -elasticsearch==6.8.1 -elasticsearch-dsl==6.1.0 email-validator==1.1.1 +Events==0.5 +exceptiongroup==1.2.1 execnet==1.7.1 -Flask==1.1.2 +executing==2.0.1 +filelock==3.15.3 +Flask==2.2.5 +Flask-Admin==1.6.1 Flask-Alembic==2.0.1 +flask-babel==4.0.0 Flask-BabelEx==0.9.4 Flask-Breadcrumbs==0.5.1 -Flask-Caching==1.9.0 -Flask-CeleryExt==0.3.4 -Flask-Cors==3.0.8 +Flask-Caching==2.3.0 +Flask-CeleryExt==0.5.0 +Flask-Collect-Invenio==1.4.0 +Flask-Cors==4.0.1 +Flask-DebugToolbar==0.15.1 Flask-KVSession-Invenio==0.6.3 Flask-Limiter==1.1.0 -Flask-Login==0.4.1 +Flask-Login==0.6.3 Flask-Mail==0.9.1 -Flask-Menu==0.7.2 +flask-menu==1.0.1 +Flask-OAuthlib==0.9.6 Flask-Principal==0.4.0 -Flask-Security==3.0.0 -flask-shell-ipython==0.4.1 -Flask-SQLAlchemy==2.4.3 -flask-talisman==0.5.0 -Flask-WTF==0.14.3 -fs==0.5.4 -ftfy==4.4.3 -future==0.18.2 +Flask-Security-Invenio==3.3.3 +flask-shell-ipython==0.5.1 +Flask-SQLAlchemy==2.5.1 +flask-talisman==0.8.1 +flask-webpackext==1.0.2 +Flask-WTF==1.2.1 +fs==2.4.16 +ftfy==6.2.0 +future==1.0.0 fuzzywuzzy==0.18.0 -html5lib==1.1 -idna==2.10 -imagesize==1.2.0 -importlib-metadata==1.7.0 -ipaddress==1.0.23 -ipython==7.16.1 -ipython-genutils==0.2.0 -isort==5.0.5 -itsdangerous==1.1.0 -jedi==0.17.1 -Jinja2==2.11.2 -jsonpatch==1.26 -jsonpointer==2.0 -jsonref==0.2 -jsonresolver==0.3.1 -jsonschema==3.2.0 -kombu==4.6.11 -limits==1.5.1 -lxml==4.5.1 -Mako==1.1.3 -MarkupSafe==1.1.1 -marshmallow==3.6.1 -maxminddb==1.5.4 +github3.py==4.0.1 +greenlet==3.0.3 +gunicorn==22.0.0 +h11==0.14.0 +idna==3.7 +imagesize==1.4.1 +importlib_metadata==7.1.0 +importlib_resources==6.4.0 +infinity==1.5 +intervals==0.9.2 +invenio-accounts==5.0.1 +invenio-app==1.3.4 +invenio-assets==3.0.3 +invenio-base==1.4.0 +invenio-cache==1.3.0 +invenio-celery==1.3.1 +invenio-config==1.0.4 +invenio-db==1.1.5 +invenio-files-rest==2.2.0 +invenio-i18n==2.1.1 +invenio-indexer==2.3.0 +invenio-jsonschemas==1.1.4 +invenio-logging==2.1.1 +invenio-mail==2.1.1 +invenio-migrator==1.0.0a10 +invenio-pidstore==1.3.1 +invenio-query-parser==0.6.0 +invenio-records==2.3.0 +invenio-records-files==1.2.1 +invenio-records-rest==2.4.1 +invenio-rest==1.3.0 +invenio-search==2.3.1 +invenio-theme==3.1.0 +ipython==8.18.1 +isort==5.13.2 +itsdangerous==2.0.1 +jedi==0.19.1 +Jinja2==3.1.4 +jsmin==3.0.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonref==1.1.0 +jsonresolver==0.3.2 +jsonschema==4.22.0 +jsonschema-specifications==2023.12.1 +kombu==5.3.7 +Levenshtein==0.25.1 +limits==1.6 +lorem==0.1.1 +lxml==5.2.2 +Mako==1.3.5 +MarkupSafe==2.1.5 +marshmallow==3.21.3 +matplotlib-inline==0.1.7 +maxminddb==2.6.2 maxminddb-geolite2==2018.703 -more-itertools==8.4.0 -msgpack==1.0.0 +more-itertools==10.3.0 +msgpack==1.0.8 +oauthlib==2.1.0 +opensearch-dsl==2.1.0 +opensearch-py==2.6.0 ordereddict==1.1 -packaging==20.4 -parso==0.7.0 -passlib==1.7.2 +outcome==1.3.0.post0 +packaging==24.1 +parso==0.8.4 +passlib==1.7.4 pathlib==1.0.1 -pep517==0.8.2 pep8==1.7.1 -pexpect==4.8.0 -pickleshare==0.7.5 +pexpect==4.9.0 +pillow==10.3.0 +pipenv==2024.0.1 +platformdirs==4.2.2 pluggy==0.13.1 -prompt-toolkit==3.0.5 -psycopg2-binary==2.8.5 -ptyprocess==0.6.0 -py==1.9.0 -pycountry==20.7.3 -pycparser==2.20 -pydocstyle==5.0.2 -Pygments==2.6.1 -PyJWT==1.7.1 -pyparsing==2.4.7 +prompt_toolkit==3.0.47 +psycopg2-binary==2.9.9 +ptyprocess==0.7.0 +pure-eval==0.2.2 +py==1.11.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pycountry==18.12.8 +pycparser==2.22 +pydocstyle==6.3.0 +Pygments==2.18.0 +PyJWT==2.8.0 +pynpm==0.2.0 +pyOpenSSL==24.1.0 pyPEG2==2.15.2 -pyrsistent==0.16.0 +pyproject_hooks==1.1.0 +PySocks==1.7.1 pytest==4.6.11 pytest-cache==1.0 -pytest-cov==2.10.0 +pytest-cov==2.10.1 pytest-flask==0.15.1 -pytest-invenio==1.3.2 +pytest-invenio==1.3.4 pytest-pep8==1.0.6 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-Levenshtein==0.12.0 -pytz==2020.1 -PyYAML==5.3.1 -redis==3.5.3 -requests==2.24.0 -selenium==3.141.0 -simplejson==3.17.0 +pytest-runner==4.5.1 +python-dateutil==2.9.0.post0 +python-geoip==1.2 +python-ldap==3.4.4 +python-Levenshtein==0.25.1 +pytz==2024.1 +pywebpack==2.0.0 +PyYAML==6.0.1 +rapidfuzz==3.9.3 +redis==5.0.6 +referencing==0.35.1 +requests==2.32.3 +requests-oauthlib==1.1.0 +rpds-py==0.18.1 +selenium==4.21.0 +sentry-sdk==1.45.0 +simplejson==3.19.2 simplekv==0.14.1 -six==1.15.0 -snowballstemmer==2.0.0 +six==1.16.0 +sniffio==1.3.1 +snowballstemmer==2.2.0 +sortedcontainers==2.4.0 speaklater==1.3 -Sphinx==3.1.2 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==1.0.3 +Sphinx==7.3.7 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.4 -SQLAlchemy==1.3.18 -SQLAlchemy-Continuum==1.3.11 -SQLAlchemy-Utils==0.35.0 -toml==0.10.1 -traitlets==4.3.3 -ua-parser==0.10.0 -uritools==3.0.0 -urllib3==1.25.9 -vine==1.3.0 -wcwidth==0.2.5 +sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-serializinghtml==1.1.10 +SQLAlchemy==1.4.52 +SQLAlchemy-Continuum==1.4.1 +SQLAlchemy-Utils==0.38.3 +stack-data==0.6.3 +tomli==2.0.1 +traitlets==5.14.3 +trio==0.25.1 +trio-websocket==0.11.1 +types-python-dateutil==2.9.0.20240316 +typing_extensions==4.12.2 +ua-parser==0.18.0 +uritemplate==4.1.1 +uritemplate.py==1.0.1 +uritools==4.0.3 +urllib3==1.26.19 +uWSGI==2.0.26 +uwsgi-tools==1.1.1 +uwsgitop==0.12 +validators==0.28.3 +vine==5.1.0 +virtualenv==20.26.2 +watchdog==2.2.1 +wcwidth==0.2.13 webargs==5.5.3 webencodings==0.5.1 -Werkzeug==1.0.1 -WTForms==2.3.1 -zipp==3.1.0 +Werkzeug==2.2.3 +wsproto==1.2.0 +WTForms==2.3.3 +WTForms-Alchemy==0.18.0 +WTForms-Components==0.10.5 +zipp==3.19.2 diff --git a/setup.py b/setup.py index e6f0043..ccd5996 100644 --- a/setup.py +++ b/setup.py @@ -45,25 +45,25 @@ ] install_requires = [ - 'arrow>=0.16', - 'six>=1.12.0', - 'cds-dojson>=0.9.0', - 'Flask-BabelEx>=0.9.3', - 'invenio-search[elasticsearch7]>=1.2.0', - 'invenio-config>=1.0.0', - 'invenio-logging>=1.0.0', - 'invenio-db[postgresql,versioning]>=1.0.0', - 'invenio-files-rest>=1.2.0', - 'invenio-migrator>=1.0.0a9', - 'invenio-pidstore>=1.0.0', - 'invenio-records>=1.0.0', - 'invenio-records-files>=1.0.0a10', + 'arrow>=1.3', + 'six>=1.16.0', + 'cds-dojson>=0.11.0', + 'Flask-BabelEx>=0.9.4', + 'invenio-search[OpenSearch2]>=2.3.1', + 'invenio-config>=1.0.4', + 'invenio-logging>=2.1.1', + 'invenio-db[postgresql,versioning]>=1.1.0', + 'invenio-files-rest>=2.2.0', + 'invenio-migrator>=1.0.0a10', + 'invenio-pidstore>=1.3.1', + 'invenio-records>=2.3.0', + 'invenio-records-files>=1.2.1', 'pathlib>=1.0.1', - 'importlib-metadata>=0.17', + 'importlib-metadata>=7.1.0', 'pluggy>=0.11.0', - 'secrets>=1.0.2', - 'fuzzywuzzy>=0.17.0', - 'python-Levenshtein>=0.12', + # 'secrets>=1.0.2', + 'fuzzywuzzy>=0.18.0', + 'python-Levenshtein>=0.25.1', # needed by legacy pip resolver ]