From a09cc008b650c5741482b60b0708a0eaa4cb9255 Mon Sep 17 00:00:00 2001 From: Osma Suominen Date: Thu, 2 Apr 2020 09:28:50 +0300 Subject: [PATCH 01/14] Switch to Travis Xenial env; require Python 3.6+ --- .travis.yml | 17 +++++++---------- README.md | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index b435a1c98..9f98a6252 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ -dist: xenial +dist: bionic language: python python: -- '3.5' - '3.6' - '3.7' - '3.8' @@ -31,18 +30,16 @@ install: # (tensorflow 2.0.0 isn't available for python 3.8 anyway) - if [[ $TRAVIS_PYTHON_VERSION != '3.8' ]]; then pip install .[nn]; fi # Install the optional Omikuji dependency -# - except for one Python version (3.6) so that we can test also without them -- if [[ $TRAVIS_PYTHON_VERSION != '3.6' ]]; then pip install .[omikuji]; fi -# For Python 3.5, also install optional dependencies that were not specified in Pipfile +# - except for one Python version (3.7) so that we can test also without them +- if [[ $TRAVIS_PYTHON_VERSION != '3.7' ]]; then pip install .[omikuji]; fi +# For Python 3.6, also install optional dependencies that were not specified in Pipfile # For other Python versions we will only run the tests that depend on pure Python modules # - fastText dependencies -- if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install fasttextmirror==0.8.22; fi +- if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then pip install .[fasttext]; fi # - voikko dependencies -- if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install voikko; fi +- if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then pip install .[voikko]; fi # - vw dependencies -- if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then sudo ln -sf /usr/lib/x86_64-linux-gnu/libboost_python-py35.a /usr/lib/x86_64-linux-gnu/libboost_python3.a; fi -- if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then sudo ln -sf /usr/lib/x86_64-linux-gnu/libboost_python-py35.so /usr/lib/x86_64-linux-gnu/libboost_python3.so; fi -- if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install vowpalwabbit==8.7.*; fi +- if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then pip install .[vw]; fi script: - pytest --cov=./ after_success: diff --git a/README.md b/README.md index 02eae8d4f..fde01c96f 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ already functional for many common tasks. # Basic install -You will need Python 3.5+ to install Annif. +You will need Python 3.6+ to install Annif. The recommended way is to install Annif from [PyPI](https://pypi.org/project/annif/) into a virtual environment. From b92e3070b7c6012ed11a6893dda353216135e14b Mon Sep 17 00:00:00 2001 From: Osma Suominen Date: Thu, 2 Apr 2020 10:02:55 +0300 Subject: [PATCH 02/14] Move dev dependencies to setup.py; remove use of pipenv --- .travis.yml | 5 +---- Pipfile | 31 ------------------------------- setup.py | 14 +++++++++++++- 3 files changed, 14 insertions(+), 36 deletions(-) delete mode 100644 Pipfile diff --git a/.travis.yml b/.travis.yml index 9f98a6252..cd952859c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,10 +20,7 @@ cache: pip before_install: - export BOTO_CONFIG=/dev/null install: -- pip install --upgrade pip -- pip install pipenv -- pip install --upgrade pytest -- pipenv install --dev --skip-lock +- pip install .[dev] - travis_wait 30 python -m nltk.downloader punkt # Install the optional neural network dependencies (TensorFlow and LMDB) # - except for one Python version (3.8) so that we can test also without them diff --git a/Pipfile b/Pipfile deleted file mode 100644 index bf2ef9502..000000000 --- a/Pipfile +++ /dev/null @@ -1,31 +0,0 @@ -[[source]] -url = "https://pypi.python.org/simple" -name = "pypi" -verify_ssl = true - -[dev-packages] -codecov = "*" -pytest-cov = "*" -pytest-watch = "*" -pytest-flask = "*" -"pytest-pep8" = "*" -swagger-tester = "*" -bumpversion = "*" -responses = "*" - -[packages] -"e1839a8" = {path = ".", editable = true} -connexion = {extras = ["swagger-ui"]} -swagger-ui-bundle = "*" -flask-cors = "*" -click = "==7.1.*" -click-log = "*" -joblib = "==0.14.*" -nltk = "*" -gensim = "==3.8.*" -scikit-learn = "==0.22.*" -rdflib = "*" -gunicorn = "!=20.0.0" -numpy = "==1.17.*" - -[requires] diff --git a/setup.py b/setup.py index ad7c435dc..ff78fcee3 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ def read(fname): 'gensim==3.8.*', 'scikit-learn==0.22.*', 'rdflib', + 'gunicorn', 'numpy==1.17.*', ], tests_require=['py', 'pytest', 'requests'], @@ -37,8 +38,19 @@ def read(fname): 'fasttext': ['fasttextmirror==0.8.22'], 'voikko': ['voikko'], 'vw': ['vowpalwabbit==8.7.*'], - 'nn': ['tensorflow==2.0.*', 'lmdb==0.98'], + 'nn': ['tensorflow==2.2.*', 'lmdb==0.98'], 'omikuji': ['omikuji==0.2.*'], + 'dev': [ + 'codecov', + 'pytest-cov', + 'pytest-watch', + 'pytest-flask', + 'pytest-pep8', + 'swagger-tester', + 'bumpversion', + 'responses', + 'autopep8' + ] }, entry_points={ 'console_scripts': ['annif=annif.cli:cli']}, From 0e7b10d120324db11b781cb0c48f5a1b5d8dec8c Mon Sep 17 00:00:00 2001 From: Osma Suominen Date: Thu, 2 Apr 2020 10:14:03 +0300 Subject: [PATCH 03/14] specify tensorflow version 2.2.0rc2 for now --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ff78fcee3..aca385781 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ def read(fname): 'fasttext': ['fasttextmirror==0.8.22'], 'voikko': ['voikko'], 'vw': ['vowpalwabbit==8.7.*'], - 'nn': ['tensorflow==2.2.*', 'lmdb==0.98'], + 'nn': ['tensorflow==2.2.0rc2', 'lmdb==0.98'], 'omikuji': ['omikuji==0.2.*'], 'dev': [ 'codecov', From 1e7b28096c36457babdfb7383c3e1575842c10b7 Mon Sep 17 00:00:00 2001 From: Osma Suominen Date: Thu, 2 Apr 2020 10:35:57 +0300 Subject: [PATCH 04/14] update README to reflect dropping Pipenv --- README.md | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index fde01c96f..a4fbf7cf6 100644 --- a/README.md +++ b/README.md @@ -52,25 +52,22 @@ for details. A development version of Annif can be installed by cloning the [GitHub repository](https://github.com/NatLibFi/Annif). -[Pipenv](https://docs.pipenv.org/) is used for managing dependencies for the -development version. ## Installation and setup Clone the repository. Switch into the repository directory. -Install pipenv if you don't have it: - pip install pipenv # or pip3 install pipenv +Create and activate a virtual environment (optional, but highly recommended): -Install dependencies and download NLTK data: + python3 -m venv venv + . venv/bin/activate - pipenv install # use --dev if you want to run tests etc. +Install dependencies (including development) and make the installation editable: -Enter the virtual environment: - - pipenv shell + pip install .[dev] + pip install -e , You will also need NLTK data files: @@ -82,7 +79,7 @@ Start up the application: ## Unit tests -Run `pipenv shell` to enter the virtual environment and then run `pytest`. +Run `. venv/bin/activate` to enter the virtual environment and then run `pytest`. To have the test suite watch for changes in code and run automatically, use pytest-watch by running `ptw`. From 2ec1b8c765d418567c3a1b236b29cc7c1c7329e3 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 2 Apr 2020 11:02:44 +0300 Subject: [PATCH 05/14] Do not send data dir to docker build context --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 766836180..6453252ac 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ **/__pycache__ **/*.pyc +data From e3e4e25dcf16fd2f0ccc2eb062e5bc80e6884148 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 2 Apr 2020 11:16:35 +0300 Subject: [PATCH 06/14] Typofix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a4fbf7cf6..0541c05cf 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Create and activate a virtual environment (optional, but highly recommended): Install dependencies (including development) and make the installation editable: pip install .[dev] - pip install -e , + pip install -e . You will also need NLTK data files: From 7694730e0f80575bf699e165eb409013b8d8e9c2 Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 2 Apr 2020 11:22:00 +0300 Subject: [PATCH 07/14] Remove use of pipenv in Dockerfile --- Dockerfile | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index c1eed5cd5..773d0767a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,26 +63,21 @@ RUN apt-get update \ && rm -rf /root/.cache/pip*/* -## Install Annif: -# Files needed by pipenv install: -COPY Pipfile README.md setup.py /Annif/ -WORKDIR /Annif +## Install Annif (including development packages and making the installation editable): -# Handle occasional timeout in nltk.downloader with 3 tries -RUN pip install pipenv --no-cache-dir \ - && pipenv install --dev --system --skip-lock \ - && for i in 1 2 3; do python -m nltk.downloader punkt -d /usr/share/nltk_data && break || sleep 1; done \ - && pip uninstall -y pipenv \ - && rm -rf /root/.cache/pip*/* +COPY setup.py README.md LICENSE.txt projects.cfg.dist /Annif/ +COPY annif /Annif/annif +COPY tests /Annif/tests +WORKDIR /Annif +RUN pip install .[dev] --no-cache-dir \ + && pip install -e . \ + # Handle occasional timeout in nltk.downloader with 3 tries + && for i in 1 2 3; do python -m nltk.downloader punkt -d /usr/share/nltk_data && break || sleep 1; done -COPY annif annif -COPY projects.cfg.dist projects.cfg.dist -COPY LICENSE.txt LICENSE.txt WORKDIR /annif-projects - # Switch user to non-root: RUN groupadd -g 998 annif_user \ && useradd -r -u 998 -g annif_user annif_user \ From e58df271992767d8334cd0e53e5b87126d607c4a Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 2 Apr 2020 11:28:18 +0300 Subject: [PATCH 08/14] Improve comment --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 773d0767a..aa872f990 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,7 +72,7 @@ WORKDIR /Annif RUN pip install .[dev] --no-cache-dir \ && pip install -e . \ - # Handle occasional timeout in nltk.downloader with 3 tries + # Download nltk data (handle occasional timeout in with 3 tries) && for i in 1 2 3; do python -m nltk.downloader punkt -d /usr/share/nltk_data && break || sleep 1; done From 226f1f86c0aab90bc03cde148738bf05f55804ac Mon Sep 17 00:00:00 2001 From: Juho Inkinen Date: Thu, 2 Apr 2020 16:07:56 +0300 Subject: [PATCH 09/14] Remove unnecessary cython install in Dockerfile Cython was at some point needed by unofficial fasttext package --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aa872f990..2505dc630 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,6 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ && pip install --no-cache-dir \ - cython \ fasttextmirror==0.8.22 \ ## Vowpal Wabbit && apt-get install -y --no-install-recommends \ From f879fb2153ba9c1286838dcebd1c1dfdb48a7927 Mon Sep 17 00:00:00 2001 From: Osma Suominen Date: Thu, 16 Apr 2020 10:53:21 +0300 Subject: [PATCH 10/14] Upgrade TensorFlow dependency: 2.2.0rc2 -> 2.2.0rc3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aca385781..0bf98ed90 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ def read(fname): 'fasttext': ['fasttextmirror==0.8.22'], 'voikko': ['voikko'], 'vw': ['vowpalwabbit==8.7.*'], - 'nn': ['tensorflow==2.2.0rc2', 'lmdb==0.98'], + 'nn': ['tensorflow==2.2.0rc3', 'lmdb==0.98'], 'omikuji': ['omikuji==0.2.*'], 'dev': [ 'codecov', From e838f757d985d64536ed07c6c84827f9ee7a56a1 Mon Sep 17 00:00:00 2001 From: Osma Suominen Date: Thu, 16 Apr 2020 11:09:37 +0300 Subject: [PATCH 11/14] Install TensorFlow 2.2.0rc3 also in Docker image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2505dc630..fbb9b8173 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,7 @@ RUN apt-get update \ && pip install --no-cache-dir \ voikko \ vowpalwabbit==8.7.* \ - tensorflow==2.0.* \ + tensorflow==2.2.0rc3 \ omikuji==0.2.* \ # For Docker healthcheck: && apt-get install -y --no-install-recommends curl \ From 229b873a22347c37bdbc35118430ff1976ed601e Mon Sep 17 00:00:00 2001 From: Osma Suominen Date: Thu, 7 May 2020 14:53:57 +0300 Subject: [PATCH 12/14] Upgrade to TensorFlow 2.2 final (also for Dockerfile) --- Dockerfile | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fbb9b8173..73be0b507 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,7 @@ RUN apt-get update \ && pip install --no-cache-dir \ voikko \ vowpalwabbit==8.7.* \ - tensorflow==2.2.0rc3 \ + tensorflow==2.2.0 \ omikuji==0.2.* \ # For Docker healthcheck: && apt-get install -y --no-install-recommends curl \ diff --git a/setup.py b/setup.py index 0bf98ed90..499050eda 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ def read(fname): 'fasttext': ['fasttextmirror==0.8.22'], 'voikko': ['voikko'], 'vw': ['vowpalwabbit==8.7.*'], - 'nn': ['tensorflow==2.2.0rc3', 'lmdb==0.98'], + 'nn': ['tensorflow==2.2.0', 'lmdb==0.98'], 'omikuji': ['omikuji==0.2.*'], 'dev': [ 'codecov', From f5416a6b3a306b23d43bd51a9818990f0c7da61c Mon Sep 17 00:00:00 2001 From: Osma Suominen Date: Thu, 7 May 2020 14:54:17 +0300 Subject: [PATCH 13/14] Ignore Python virtual environment directory when building docker images --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 6453252ac..d0508a2da 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ **/__pycache__ **/*.pyc data +venv From e35b59a5332f29845a2058a30e096c33bde401ec Mon Sep 17 00:00:00 2001 From: Osma Suominen Date: Fri, 8 May 2020 09:53:45 +0300 Subject: [PATCH 14/14] Update the comments in .travis.yml to better match the current test setup --- .travis.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index cd952859c..0ecdc6819 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,15 +22,14 @@ before_install: install: - pip install .[dev] - travis_wait 30 python -m nltk.downloader punkt +# Selectively install the optional dependencies for some Python versions # Install the optional neural network dependencies (TensorFlow and LMDB) -# - except for one Python version (3.8) so that we can test also without them -# (tensorflow 2.0.0 isn't available for python 3.8 anyway) -- if [[ $TRAVIS_PYTHON_VERSION != '3.8' ]]; then pip install .[nn]; fi +# - except for one Python version (3.7) so that we can test also without them +- if [[ $TRAVIS_PYTHON_VERSION != '3.7' ]]; then pip install .[nn]; fi # Install the optional Omikuji dependency # - except for one Python version (3.7) so that we can test also without them - if [[ $TRAVIS_PYTHON_VERSION != '3.7' ]]; then pip install .[omikuji]; fi -# For Python 3.6, also install optional dependencies that were not specified in Pipfile -# For other Python versions we will only run the tests that depend on pure Python modules +# For Python 3.6, install the optional dependencies that depend on system libraries # - fastText dependencies - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then pip install .[fasttext]; fi # - voikko dependencies