diff --git a/.coveragerc b/.coveragerc index 895e291..59a7c1e 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,3 +3,4 @@ omit = */site-packages/* */python?.?/* ckan/* + ckanext/envvars/tests/* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5b15fff..55d2dc7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.8' - name: Install requirements run: pip install flake8 pycodestyle - name: Check syntax @@ -17,7 +17,7 @@ jobs: needs: lint strategy: matrix: - ckan-version: [2.9, 2.9-py2, 2.8, 2.7] + ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7] fail-fast: false name: CKAN ${{ matrix.ckan-version }} @@ -42,6 +42,8 @@ jobs: CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test CKAN_SOLR_URL: http://solr:8983/solr/ckan CKAN_REDIS_URL: redis://redis:6379/1 + # required for CKAN 2.10 (if datastore is enabled) + CKAN__DATAPUSHER__API_TOKEN: XX steps: - uses: actions/checkout@v3 @@ -59,8 +61,16 @@ jobs: if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }} run: | paster --plugin=ckan db init -c test.ini + - name: Run tests - run: pytest --ckan-ini=test.ini --cov=ckanext.envvars --cov-report=xml --cov-append --disable-warnings ckanext/envvars/tests.py + run: | + pytest -v --ckan-ini=test.ini \ + --cov=ckanext.envvars \ + --cov-report=xml \ + --cov-append \ + --disable-warnings \ + ckanext/envvars/tests/ + - name: Upload coverage report to codecov uses: codecov/codecov-action@v1 with: diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 42cfed1..0000000 --- a/.travis.yml +++ /dev/null @@ -1,13 +0,0 @@ -language: python -python: - - "2.7" -env: - - PGVERSION=9.1 CKANVERSION=master - - PGVERSION=9.1 CKANVERSION=2.3 -install: - - bash bin/travis-build.bash - - pip install coveralls -script: sh bin/travis-run.sh -after_success: - - coveralls -sudo: required diff --git a/README.md b/README.md index 31f8691..2c53486 100755 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ e.g.: Requirements ------------ -Tested in CKAN 2.7, CKAN 2.8 and CKAN 2.9, but may work in other +Tested in CKAN 2.7, CKAN 2.8, CKAN 2.9 and CKAN 2.10, but may work in other versions. To ensure all config settings are overridden by env var values, diff --git a/bin/travis-build.bash b/bin/travis-build.bash deleted file mode 100644 index 5b151bd..0000000 --- a/bin/travis-build.bash +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -e - -echo "This is travis-build.bash..." - -echo "Installing the packages that CKAN requires..." -sudo apt-get update -qq -sudo apt-get install postgresql-$PGVERSION solr-jetty libcommons-fileupload-java:amd64=1.2.2-1 - -echo "Installing CKAN and its Python dependencies..." -git clone https://github.com/ckan/ckan -cd ckan -if [ $ CKANVERSION == '2.3'] -then - git checkout release-v2.3 -else - git checkout master -fi -python setup.py develop -pip install -r requirements.txt --allow-all-external -pip install -r dev-requirements.txt --allow-all-external -cd - - -echo "Creating the PostgreSQL user and database..." -sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';" -sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;' - -echo "Initialising the database..." -cd ckan -paster db init -c test-core.ini -cd - - -echo "Installing ckanext-envvars and its requirements..." -python setup.py develop -pip install -r dev-requirements.txt - -echo "Moving test.ini into a subdir..." -mkdir subdir -mv test.ini subdir - -echo "travis-build.bash is done." diff --git a/bin/travis-run.sh b/bin/travis-run.sh deleted file mode 100644 index d31158e..0000000 --- a/bin/travis-run.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -e - -echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty -sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml -sudo service jetty restart -nosetests --ckan --nologcapture --with-pylons=subdir/test.ini --with-coverage --cover-package=ckanext.envvars --cover-inclusive --cover-erase --cover-tests diff --git a/ckanext/envvars/plugin.py b/ckanext/envvars/plugin.py index f97b150..87b0f02 100644 --- a/ckanext/envvars/plugin.py +++ b/ckanext/envvars/plugin.py @@ -1,8 +1,9 @@ +import logging import os - import ckan.plugins as plugins +import ckantoolkit as toolkit + -import logging log = logging.getLogger(__name__) @@ -35,3 +36,8 @@ def update_config(self, config): # override config settings with new values config.update(dict(ckan_vars)) + + # CKAN >=2.10 normalizes config values + if toolkit.check_ckan_version(min_version='2.10'): + from ckan.common import config_declaration + config_declaration.normalize(config) diff --git a/ckanext/envvars/tests/__init__.py b/ckanext/envvars/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ckanext/envvars/tests.py b/ckanext/envvars/tests/test_base_envvars.py similarity index 60% rename from ckanext/envvars/tests.py rename to ckanext/envvars/tests/test_base_envvars.py index f5b0707..0a33d9f 100644 --- a/ckanext/envvars/tests.py +++ b/ckanext/envvars/tests/test_base_envvars.py @@ -1,12 +1,13 @@ import os -import pytest - import ckan.plugins as p +import ckantoolkit as tk +from ckanext.envvars.plugin import EnvvarsPlugin -from ckantoolkit import config -import ckantoolkit as toolkit -from ckanext.envvars.plugin import EnvvarsPlugin +if tk.check_ckan_version(min_version='2.10'): + from unittest import mock +else: + import mock class TestEnvVarToIni(object): @@ -59,11 +60,11 @@ def test_envvars_values_in_config(self): self._setup_env_vars(envvar_to_ini_examples) - assert config['ckan.site_id'] == 'my-envvar-site' - assert config['ckanext.extension_setting'] == 'my-extension-value' - assert config['ckanext.another.ext_setting'] == 'my-other-extension-value' - assert config['beaker.session.key'] == 'my-beaker-key' - assert config['cache_dir'] == '/cache_directory_path/' + assert tk.config['ckan.site_id'] == 'my-envvar-site' + assert tk.config['ckanext.extension_setting'] == 'my-extension-value' + assert tk.config['ckanext.another.ext_setting'] == 'my-other-extension-value' + assert tk.config['beaker.session.key'] == 'my-beaker-key' + assert tk.config['cache_dir'] == '/cache_directory_path/' self._teardown_env_vars(envvar_to_ini_examples) @@ -88,45 +89,56 @@ def _teardown_env_vars(self, envvar_list): # plugin.load() will force the config to update p.load() - def test_core_ckan_envvar_values_in_config(self): - - if not toolkit.check_ckan_version('2.4.0'): - raise pytest.skip('CKAN version 2.4 or above needed') + # The datastore plugin, only for CKAN 2.10+, will try to + # connect to the database when it is loaded. + @mock.patch('ckanext.datastore.plugin.DatastorePlugin.configure') + def test_core_ckan_envvar_values_in_config(self, datastore_configure): core_ckan_env_var_list = [ ('CKAN_SQLALCHEMY_URL', 'postgresql://mynewsqlurl/'), - ('CKAN_DATASTORE_WRITE_URL', 'http://mynewdbwriteurl/'), - ('CKAN_DATASTORE_READ_URL', 'http://mynewdbreadurl/'), + ('CKAN_DATASTORE_WRITE_URL', 'postgresql://mynewdbwriteurl/'), + ('CKAN_DATASTORE_READ_URL', 'postgresql://mynewdbreadurl/'), ('CKAN_SITE_ID', 'my-site'), ('CKAN_DB', 'postgresql://mydeprectatesqlurl/'), + # SMTP settings takes precedence from CKAN core CONFIG_FROM_ENV_VARS ('CKAN_SMTP_SERVER', 'mail.example.com'), ('CKAN_SMTP_STARTTLS', 'True'), ('CKAN_SMTP_USER', 'my_user'), ('CKAN_SMTP_PASSWORD', 'password'), - ('CKAN_SMTP_MAIL_FROM', 'server@example.com') + + ('CKAN_SMTP_MAIL_FROM', 'server@example.com'), + ('CKAN__DATASETS_PER_PAGE', '14'), + ('CKAN__HIDE_ACTIVITY_FROM_USERS', 'user1 user2'), ] self._setup_env_vars(core_ckan_env_var_list) - assert config['sqlalchemy.url'] == 'postgresql://mynewsqlurl/' - assert config['ckan.datastore.write_url'] == 'http://mynewdbwriteurl/' - assert config['ckan.datastore.read_url'] == 'http://mynewdbreadurl/' - assert config['ckan.site_id'] == 'my-site' - assert config['smtp.server'] == 'mail.example.com' - assert config['smtp.starttls'] == 'True' - assert config['smtp.user'] == 'my_user' - assert config['smtp.password'] == 'password' - assert config['smtp.mail_from'] == 'server@example.com' + assert tk.config['sqlalchemy.url'] == 'postgresql://mynewsqlurl/' + assert tk.config['ckan.datastore.write_url'] == 'postgresql://mynewdbwriteurl/' + assert tk.config['ckan.datastore.read_url'] == 'postgresql://mynewdbreadurl/' + assert tk.config['ckan.site_id'] == 'my-site' + assert tk.config['smtp.server'] == 'mail.example.com' + + assert tk.config['smtp.user'] == 'my_user' + assert tk.config['smtp.password'] == 'password' + assert tk.config['smtp.mail_from'] == 'server@example.com' + # See https://github.com/ckan/ckan/pull/7502 + assert tk.config['smtp.starttls'] == 'True' + + if tk.check_ckan_version(min_version='2.10'): + assert tk.config['ckan.datasets_per_page'] == 14 + assert tk.config['ckan.hide_activity_from_users'] == ['user1', 'user2'] + else: + assert tk.config['ckan.datasets_per_page'] == '14' + assert tk.config['ckan.hide_activity_from_users'] == 'user1 user2' self._teardown_env_vars(core_ckan_env_var_list) - def test_core_ckan_envvar_values_in_config_take_precedence(self): + @mock.patch('ckanext.datastore.plugin.DatastorePlugin.configure') + def test_core_ckan_envvar_values_in_config_take_precedence(self, datastore_configure): '''Core CKAN env var transformations take precedence over this extension''' - if not toolkit.check_ckan_version('2.4.0'): - raise pytest.skip('CKAN version 2.4 or above needed') - combined_list = [ ('CKAN___SQLALCHEMY__URL', 'postgresql://thisextensionformat/'), ('CKAN_SQLALCHEMY_URL', 'postgresql://coreckanformat/'), @@ -134,6 +146,6 @@ def test_core_ckan_envvar_values_in_config_take_precedence(self): self._setup_env_vars(combined_list) - assert config['sqlalchemy.url'] == 'postgresql://coreckanformat/' + assert tk.config['sqlalchemy.url'] == 'postgresql://coreckanformat/' self._teardown_env_vars(combined_list) diff --git a/setup.py b/setup.py index cc3392a..0fdf894 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='0.0.2', + version='0.0.3', description='''CKAN configuration settings available from env vars''', long_description=long_description, @@ -35,7 +35,7 @@ # 3 - Alpha # 4 - Beta # 5 - Production/Stable - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', # Pick your license as you wish (should match "license" above) 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',