diff --git a/.dockerignore b/.dockerignore index 766836180..d0508a2da 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,4 @@ **/__pycache__ **/*.pyc +data +venv diff --git a/.travis.yml b/.travis.yml index b435a1c98..0ecdc6819 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ -dist: xenial +dist: bionic language: python python: -- '3.5' - '3.6' - '3.7' - '3.8' @@ -21,28 +20,22 @@ cache: pip before_install: - export BOTO_CONFIG=/dev/null install: -- pip install --upgrade pip -- pip install pipenv -- pip install --upgrade pytest -- pipenv install --dev --skip-lock +- pip install .[dev] - travis_wait 30 python -m nltk.downloader punkt +# Selectively install the optional dependencies for some Python versions # Install the optional neural network dependencies (TensorFlow and LMDB) -# - except for one Python version (3.8) so that we can test also without them -# (tensorflow 2.0.0 isn't available for python 3.8 anyway) -- if [[ $TRAVIS_PYTHON_VERSION != '3.8' ]]; then pip install .[nn]; fi +# - except for one Python version (3.7) so that we can test also without them +- if [[ $TRAVIS_PYTHON_VERSION != '3.7' ]]; then pip install .[nn]; fi # Install the optional Omikuji dependency -# - except for one Python version (3.6) so that we can test also without them -- if [[ $TRAVIS_PYTHON_VERSION != '3.6' ]]; then pip install .[omikuji]; fi -# For Python 3.5, also install optional dependencies that were not specified in Pipfile -# For other Python versions we will only run the tests that depend on pure Python modules +# - except for one Python version (3.7) so that we can test also without them +- if [[ $TRAVIS_PYTHON_VERSION != '3.7' ]]; then pip install .[omikuji]; fi +# For Python 3.6, install the optional dependencies that depend on system libraries # - fastText dependencies -- if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install fasttextmirror==0.8.22; fi +- if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then pip install .[fasttext]; fi # - voikko dependencies -- if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install voikko; fi +- if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then pip install .[voikko]; fi # - vw dependencies -- if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then sudo ln -sf /usr/lib/x86_64-linux-gnu/libboost_python-py35.a /usr/lib/x86_64-linux-gnu/libboost_python3.a; fi -- if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then sudo ln -sf /usr/lib/x86_64-linux-gnu/libboost_python-py35.so /usr/lib/x86_64-linux-gnu/libboost_python3.so; fi -- if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install vowpalwabbit==8.7.*; fi +- if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then pip install .[vw]; fi script: - pytest --cov=./ after_success: diff --git a/Dockerfile b/Dockerfile index c1eed5cd5..73be0b507 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,6 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ && pip install --no-cache-dir \ - cython \ fasttextmirror==0.8.22 \ ## Vowpal Wabbit && apt-get install -y --no-install-recommends \ @@ -54,7 +53,7 @@ RUN apt-get update \ && pip install --no-cache-dir \ voikko \ vowpalwabbit==8.7.* \ - tensorflow==2.0.* \ + tensorflow==2.2.0 \ omikuji==0.2.* \ # For Docker healthcheck: && apt-get install -y --no-install-recommends curl \ @@ -63,26 +62,21 @@ RUN apt-get update \ && rm -rf /root/.cache/pip*/* -## Install Annif: -# Files needed by pipenv install: -COPY Pipfile README.md setup.py /Annif/ -WORKDIR /Annif +## Install Annif (including development packages and making the installation editable): -# Handle occasional timeout in nltk.downloader with 3 tries -RUN pip install pipenv --no-cache-dir \ - && pipenv install --dev --system --skip-lock \ - && for i in 1 2 3; do python -m nltk.downloader punkt -d /usr/share/nltk_data && break || sleep 1; done \ - && pip uninstall -y pipenv \ - && rm -rf /root/.cache/pip*/* +COPY setup.py README.md LICENSE.txt projects.cfg.dist /Annif/ +COPY annif /Annif/annif +COPY tests /Annif/tests +WORKDIR /Annif +RUN pip install .[dev] --no-cache-dir \ + && pip install -e . \ + # Download nltk data (handle occasional timeout in with 3 tries) + && for i in 1 2 3; do python -m nltk.downloader punkt -d /usr/share/nltk_data && break || sleep 1; done -COPY annif annif -COPY projects.cfg.dist projects.cfg.dist -COPY LICENSE.txt LICENSE.txt WORKDIR /annif-projects - # Switch user to non-root: RUN groupadd -g 998 annif_user \ && useradd -r -u 998 -g annif_user annif_user \ diff --git a/Pipfile b/Pipfile deleted file mode 100644 index bf2ef9502..000000000 --- a/Pipfile +++ /dev/null @@ -1,31 +0,0 @@ -[[source]] -url = "https://pypi.python.org/simple" -name = "pypi" -verify_ssl = true - -[dev-packages] -codecov = "*" -pytest-cov = "*" -pytest-watch = "*" -pytest-flask = "*" -"pytest-pep8" = "*" -swagger-tester = "*" -bumpversion = "*" -responses = "*" - -[packages] -"e1839a8" = {path = ".", editable = true} -connexion = {extras = ["swagger-ui"]} -swagger-ui-bundle = "*" -flask-cors = "*" -click = "==7.1.*" -click-log = "*" -joblib = "==0.14.*" -nltk = "*" -gensim = "==3.8.*" -scikit-learn = "==0.22.*" -rdflib = "*" -gunicorn = "!=20.0.0" -numpy = "==1.17.*" - -[requires] diff --git a/README.md b/README.md index 02eae8d4f..0541c05cf 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ already functional for many common tasks. # Basic install -You will need Python 3.5+ to install Annif. +You will need Python 3.6+ to install Annif. The recommended way is to install Annif from [PyPI](https://pypi.org/project/annif/) into a virtual environment. @@ -52,25 +52,22 @@ for details. A development version of Annif can be installed by cloning the [GitHub repository](https://github.com/NatLibFi/Annif). -[Pipenv](https://docs.pipenv.org/) is used for managing dependencies for the -development version. ## Installation and setup Clone the repository. Switch into the repository directory. -Install pipenv if you don't have it: - pip install pipenv # or pip3 install pipenv +Create and activate a virtual environment (optional, but highly recommended): -Install dependencies and download NLTK data: + python3 -m venv venv + . venv/bin/activate - pipenv install # use --dev if you want to run tests etc. +Install dependencies (including development) and make the installation editable: -Enter the virtual environment: - - pipenv shell + pip install .[dev] + pip install -e . You will also need NLTK data files: @@ -82,7 +79,7 @@ Start up the application: ## Unit tests -Run `pipenv shell` to enter the virtual environment and then run `pytest`. +Run `. venv/bin/activate` to enter the virtual environment and then run `pytest`. To have the test suite watch for changes in code and run automatically, use pytest-watch by running `ptw`. diff --git a/setup.py b/setup.py index ad7c435dc..499050eda 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ def read(fname): 'gensim==3.8.*', 'scikit-learn==0.22.*', 'rdflib', + 'gunicorn', 'numpy==1.17.*', ], tests_require=['py', 'pytest', 'requests'], @@ -37,8 +38,19 @@ def read(fname): 'fasttext': ['fasttextmirror==0.8.22'], 'voikko': ['voikko'], 'vw': ['vowpalwabbit==8.7.*'], - 'nn': ['tensorflow==2.0.*', 'lmdb==0.98'], + 'nn': ['tensorflow==2.2.0', 'lmdb==0.98'], 'omikuji': ['omikuji==0.2.*'], + 'dev': [ + 'codecov', + 'pytest-cov', + 'pytest-watch', + 'pytest-flask', + 'pytest-pep8', + 'swagger-tester', + 'bumpversion', + 'responses', + 'autopep8' + ] }, entry_points={ 'console_scripts': ['annif=annif.cli:cli']},