From ee6930d17fecab3e393a1596b2f6f08d13102097 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Mon, 16 Dec 2024 14:01:35 +1000 Subject: [PATCH] [QOLDEV-1014] sync test config with other repos - Add report-only testing of CKAN 2.11 - Drop Python 2 support - Replace obsolete test containers - Add more regression scenario tests --- .ahoy.yml | 23 +- .docker/Dockerfile-template.ckan | 33 +- .docker/test.ini | 22 +- .github/workflows/test.yml | 24 +- README.md | 10 +- bin/build.sh | 15 +- bin/create-test-data.sh | 99 +- bin/docker-compose.sh | 10 +- bin/init-ext.sh | 10 +- bin/init.sh | 2 +- bin/process-artifacts.sh | 5 +- bin/serve.sh | 16 +- .../harvester_data_qld_geoscience/plugin.py | 5 +- dev-requirements-2.9-py2.txt | 34 - dev-requirements-2.9.txt | 30 +- dev-requirements.txt | 31 +- docker-compose.yml | 4 +- test/features/comments.feature | 10 + test/features/data_qld_theme.feature | 18 +- test/features/data_usability_rating.feature | 4 +- test/features/datasets.feature | 8 +- test/features/organisations.feature | 3 + .../features/resource_type_validation.feature | 16 + test/features/search_facets.feature | 2 + test/features/steps/steps.py | 31 +- test/features/users.feature | 28 + test/features/xloader.feature | 36 + test/fixtures/eicar.com.pdf | Bin 0 -> 68 bytes test/fixtures/test_game_data.csv | 900 ------------------ 29 files changed, 293 insertions(+), 1136 deletions(-) delete mode 100644 dev-requirements-2.9-py2.txt create mode 100644 test/features/resource_type_validation.feature create mode 100644 test/features/xloader.feature create mode 100644 test/fixtures/eicar.com.pdf diff --git a/.ahoy.yml b/.ahoy.yml index 6cfa604..9e5dafd 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -38,7 +38,7 @@ commands: ahoy title "Building and starting Docker containers" sh bin/docker-compose.sh up -d "$@" echo "Initialising database schema" - ahoy cli '$APP_DIR/bin/init.sh' + ahoy cli '"${APP_DIR}"/bin/init.sh' echo "Waiting for containers to start listening..." ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m" if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi @@ -79,9 +79,9 @@ commands: cmd: | CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan) if [ "${#}" -ne 0 \]; then - docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*" + docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate; cd $APP_DIR;'" $*" else - docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh' + docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate && cd $APP_DIR && sh' fi doctor: @@ -92,7 +92,7 @@ commands: usage: Install test site data. cmd: | ahoy title "Installing a fresh site" - ahoy cli '$APP_DIR/bin/init.sh && $APP_DIR/bin/create-test-data.sh' + ahoy cli '"${APP_DIR}"/bin/init.sh && "${APP_DIR}"/bin/create-test-data.sh' clean: usage: Remove containers and all build files. @@ -128,13 +128,13 @@ commands: cmd: | docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/ docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/ - ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI; $APP_DIR/bin/process-config.sh' + ahoy cli 'chmod -v u+x /usr/bin/ckan_cli "${APP_DIR}"/bin/*; cp -v .docker/test.ini $CKAN_INI; "${APP_DIR}"/bin/process-config.sh' test-unit: usage: Run unit tests. cmd: | ahoy title 'Run unit tests' - ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/ckanext' || \ + ahoy cli 'pytest --ckan-ini=${CKAN_INI} --cov=ckanext "${APP_DIR}"/ckanext --junit-xml=test/junit/results.xml' || \ [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ] test-bdd: @@ -145,21 +145,22 @@ commands: ahoy start-ckan-job-workers ahoy start-mailmock & sleep 5 + JUNIT_OUTPUT="--junit --junit-directory=test/junit/" if [ "$BEHAVE_TAG" = "" ]; then # no tag specified, probably running locally - (ahoy cli "behave -k ${*:-test/features} --tags=smoke" \ - && ahoy cli "behave -k ${*:-test/features} --tags=-smoke" \ + (ahoy cli "behave $JUNIT_OUTPUT -k ${*:-test/features} --tags=smoke" \ + && ahoy cli "behave $JUNIT_OUTPUT -k ${*:-test/features} --tags=-smoke" ) || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] elif [ "$BEHAVE_TAG" = "authenticated" ]; then # run any tests that don't have a specific tag - ahoy cli "behave -k ${*:-test/features} --tags=-unauthenticated --tags=-smoke --tags=-OpenData --tags=-multi_plugin" \ + ahoy cli "behave $JUNIT_OUTPUT -k ${*:-test/features} --tags=-unauthenticated --tags=-smoke --tags=-OpenData --tags=-multi_plugin" \ || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] else if [ "$BEHAVE_TAG" != "multi_plugin" ]; then BEHAVE_TAG="$BEHAVE_TAG --tags=-multi_plugin" fi # run tests with the specified tag - ahoy cli "behave -k ${*:-test/features} --tags=$BEHAVE_TAG" \ + ahoy cli "behave $JUNIT_OUTPUT -k ${*:-test/features} --tags=$BEHAVE_TAG" \ || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] fi ahoy stop-mailmock @@ -169,7 +170,7 @@ commands: usage: Starts email mock server used for email BDD tests cmd: | ahoy title 'Starting mailmock' - ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails' # for debugging mailmock email output remove --no-stdout + ahoy cli 'mailmock -p 8025 -o "${APP_DIR}"/test/emails' # for debugging mailmock email output remove --no-stdout stop-mailmock: usage: Stops email mock server used for email BDD tests diff --git a/.docker/Dockerfile-template.ckan b/.docker/Dockerfile-template.ckan index 53acf91..73247aa 100644 --- a/.docker/Dockerfile-template.ckan +++ b/.docker/Dockerfile-template.ckan @@ -1,4 +1,8 @@ -FROM openknowledge/ckan-dev:{CKAN_VERSION} +FROM ckan/ckan-dev:{CKAN_VERSION} + +# swap between root and unprivileged user +ARG ORIGINAL_USER +RUN ORIGINAL_USER=$(id -un) ARG SITE_URL=http://ckan:5000/ ENV PYTHON_VERSION={PYTHON_VERSION} @@ -8,11 +12,22 @@ ENV PYTHON={PYTHON} WORKDIR "${APP_DIR}" -ENV DOCKERIZE_VERSION v0.6.1 -RUN apk add --no-cache build-base \ - && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ +COPY .docker/test.ini $CKAN_INI + +COPY . "${APP_DIR}"/ + +USER root + +COPY bin/ckan_cli /usr/bin/ + +RUN chmod +x "${APP_DIR}"/bin/*.sh /usr/bin/ckan_cli + +ENV DOCKERIZE_VERSION=v0.6.1 +RUN wget -O - https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ | tar -C /usr/local/bin -xzvf - +RUN which ps || apt-get install -y procps + # Install CKAN. RUN cd $SRC_DIR/ckan \ @@ -22,15 +37,9 @@ RUN cd $SRC_DIR/ckan \ && git reset --hard && git clean -f \ && git checkout '{CKAN_GIT_VERSION}' -COPY .docker/test.ini $CKAN_INI - -COPY . ${APP_DIR}/ - -COPY bin/ckan_cli /usr/bin/ - -RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli +USER "$ORIGINAL_USER" # Init current extension. -RUN ${APP_DIR}/bin/init-ext.sh +RUN "${APP_DIR}"/bin/init-ext.sh CMD ["/srv/app/bin/serve.sh"] diff --git a/.docker/test.ini b/.docker/test.ini index b630b21..23b7c20 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -1,16 +1,3 @@ -# -# CKAN - Pylons configuration -# -# These are some of the configuration options available for your CKAN -# instance. Check the documentation in 'doc/configuration.rst' or at the -# following URL for a description of what they do and the full list of -# available options: -# -# http://docs.ckan.org/en/latest/maintaining/configuration.html -# -# The %(here)s variable will be replaced with the parent directory of this file -# - [DEFAULT] debug = false smtp_server = localhost:8025 @@ -30,13 +17,9 @@ full_stack = true cache_dir = /tmp/%(ckan.site_id)s/ beaker.session.key = ckan -# This is the secret token that the beaker library uses to hash the cookie sent -# to the client. `paster make-config` generates a unique value for this each -# time it generates a config file. +SECRET_KEY = bSmgPpaxg2M+ZRes3u1TXwIcE beaker.session.secret = bSmgPpaxg2M+ZRes3u1TXwIcE -# `paster make-config` generates a unique value for this each time it generates -# a config file. app_instance_uuid = 6e3daf8e-1c6b-443b-911f-c7ab4c5f9605 # repoze.who config @@ -71,6 +54,7 @@ ckan.auth.create_user_via_api = false ckan.auth.create_user_via_web = true ckan.auth.roles_that_cascade_to_sub_groups = admin ckan.auth.public_user_details = False +ckan.auth.reveal_private_datasets = True ## Search Settings @@ -250,7 +234,7 @@ ckanext-archiver.cache_url_root = http://dataqld-ckan.docker.amazee.io/resources # QA qa.resource_format_openness_scores_json = /srv/app/src/ckanext-data-qld/ckanext/data_qld/resource_format_openness_scores.json -# Logging configuration +## Logging configuration [loggers] keys = root, ckan, ckanext diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b8a1ce..f6c5454 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,12 @@ jobs: matrix: ckan-version: ["2.10", 2.9] behave-tag: [smoke, unauthenticated, multi_plugin, OpenData, authenticated] + experimental: [false] + include: + - ckan-version: '2.11' + experimental: true + - ckan-version: 'master' + experimental: true #master is unstable, good to know if we are compatible or not name: Run ${{ matrix.behave-tag }} tests on CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest @@ -42,34 +48,48 @@ jobs: run: echo HOME=/root >> "$GITHUB_ENV" - uses: actions/checkout@v4 + continue-on-error: ${{ matrix.experimental }} timeout-minutes: 2 - name: Build + continue-on-error: ${{ matrix.experimental }} run: bin/build.sh timeout-minutes: 15 - name: Unit tests + continue-on-error: ${{ matrix.experimental }} if: ${{ matrix.behave-tag == 'smoke' }} run: bin/test.sh timeout-minutes: 15 - name: Test ${{ matrix.behave-tag }} BDD + continue-on-error: ${{ matrix.experimental }} run: bin/test-bdd.sh timeout-minutes: 45 - name: Retrieve logs if: always() run: ahoy logs + continue-on-error: ${{ matrix.experimental }} timeout-minutes: 1 - - name: Retrieve screenshots + - name: Retrieve results if: always() run: bin/process-artifacts.sh + continue-on-error: ${{ matrix.experimental }} timeout-minutes: 1 + - name: Test Summary + uses: test-summary/action@v2 + continue-on-error: ${{ matrix.experimental }} + with: + paths: "/tmp/artifacts/junit/*.xml" + if: always() + - name: Upload screenshots - if: failure() + if: always() uses: actions/upload-artifact@v4 + continue-on-error: ${{ matrix.experimental }} with: name: CKAN ${{ matrix.ckan-version }} ${{ matrix.behave-tag }} screenshots path: /tmp/artifacts/behave/screenshots diff --git a/README.md b/README.md index 6d86cc6..77ae7bc 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,13 @@ ckan.plugins = harvest harvester_data_qld_geoscience - [Pygmy](https://pygmy.readthedocs.io/) - [Ahoy](https://github.com/ahoy-cli/ahoy) -- Build the test container for your preferred CKAN version: '2.8', '2.9-py2', or '2.9'. +- Build the test container for your preferred CKAN version: '2.9' or '2.10'. ``` -CKAN_VERSION=2.9 .circleci/build.sh +CKAN_VERSION=2.10 bin/build.sh ``` -- Run tests: `.circleci/test.sh` +- Run tests: +``` +bin/test.sh +bin/test.sh +``` diff --git a/bin/build.sh b/bin/build.sh index 3c907b0..9340451 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -14,23 +14,18 @@ sed -i -e "s/##//" docker-compose.yml # Pull the latest images. ahoy pull -PYTHON=python +PYTHON=python3 +PYTHON_VERSION=py3 CKAN_GIT_VERSION=$CKAN_VERSION CKAN_GIT_ORG=qld-gov-au -if [ "$CKAN_VERSION" = "2.10" ]; then +if [ "$CKAN_VERSION" = "2.11" ]; then + CKAN_GIT_VERSION=ckan-2.11.1 +elif [ "$CKAN_VERSION" = "2.10" ]; then CKAN_GIT_VERSION=ckan-2.10.5-qgov.4 - PYTHON_VERSION=py3 - PYTHON="${PYTHON}3" else CKAN_GIT_VERSION=ckan-2.9.9-qgov.3 - if [ "$CKAN_VERSION" = "2.9-py2" ]; then - PYTHON_VERSION=py2 - else - PYTHON_VERSION=py3 - PYTHON="${PYTHON}3" - fi fi sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \ diff --git a/bin/create-test-data.sh b/bin/create-test-data.sh index a34e671..10bc7bf 100644 --- a/bin/create-test-data.sh +++ b/bin/create-test-data.sh @@ -9,7 +9,7 @@ CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" -. ${APP_DIR}/bin/activate +. "${APP_DIR}"/bin/activate add_user_if_needed () { echo "Adding user '$2' ($1) with email address [$3]" @@ -19,6 +19,10 @@ add_user_if_needed () { password="${4:-Password123!}" } +api_call () { + wget -O - --header="Authorization: ${API_KEY}" --post-data "$1" ${CKAN_ACTION_URL}/$2 +} + add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL" ckan_cli sysadmin add "${CKAN_USER_NAME}" @@ -36,15 +40,12 @@ sed -i "s/{API_TOKEN}/$API_KEY/" $CKAN_INI # echo "Adding sysadmin config:" -curl -LsH "Authorization: ${API_KEY}" \ - --header "Content-Type: application/json" \ - --data '{ +api_call '{ "ckan.comments.profanity_list": "", "ckan.datarequests.closing_circumstances": "Released as open data|nominate_dataset\r\nOpen dataset already exists|nominate_dataset\r\nPartially released|nominate_dataset\r\nTo be released as open data at a later date|nominate_approximate_date\r\nData openly available elsewhere\r\nNot suitable for release as open data\r\nRequested data not available/cannot be compiled\r\nRequestor initiated closure", "ckanext.data_qld.resource_formats": "CSV\r\nHTML\r\nJSON\r\nRDF\r\nTXT\r\nXLS", "ckanext.data_qld.excluded_display_name_words": "gov" - }' \ - ${CKAN_ACTION_URL}/config_option_update + }' config_option_update ## # END. @@ -66,27 +67,20 @@ add_user_if_needed test_org_member "Test Member" test_org_member@localhost echo "Creating ${TEST_ORG_TITLE} organisation:" TEST_ORG=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'", - "description": "Organisation for testing issues"}' \ - ${CKAN_ACTION_URL}/organization_create + api_call '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'", + "description": "Organisation for testing issues"}' organization_create ) -TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py) +TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON "${APP_DIR}"/bin/extract-id.py) echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \ - ${CKAN_ACTION_URL}/member_create +api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' member_create + +api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' member_create -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' \ - ${CKAN_ACTION_URL}/member_create +api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' member_create -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' \ - ${CKAN_ACTION_URL}/member_create ## # END. # @@ -94,9 +88,7 @@ curl -LsH "Authorization: ${API_KEY}" \ # Creating test data hierarchy which creates organisations assigned to datasets echo "Creating food-standards-agency organisation:" organisation_create=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data "name=food-standards-agency&title=Food%20Standards%20Agency" \ - ${CKAN_ACTION_URL}/organization_create + api_call "name=food-standards-agency&title=Food%20Standards%20Agency" organization_create ) echo ${organisation_create} @@ -104,24 +96,21 @@ add_user_if_needed group_admin "Group Admin" group_admin@localhost add_user_if_needed walker "Walker" walker@localhost # Create private test dataset with our standard fields -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true, +api_call '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true, "update_frequency": "monthly", "author_email": "admin@localhost", "version": "1.0", "license_id": "other-open", "data_driven_application": "NO", "security_classification": "PUBLIC", -"notes": "private test", "de_identified_data": "NO"}' \ - ${CKAN_ACTION_URL}/package_create +"notes": "private test", "de_identified_data": "NO"}' package_create # Create public test dataset with our standard fields -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", +api_call '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "update_frequency": "monthly", "author_email": "admin@example.com", "version": "1.0", "license_id": "other-open", "data_driven_application": "NO", "security_classification": "PUBLIC", "notes": "public test", "de_identified_data": "NO", "resources": [ {"name": "test-resource", "description": "Test resource description", "url": "https://example.com/foo", "format": "HTML", "size": 1024} -]}' \ - ${CKAN_ACTION_URL}/package_create +]}' package_create +# Populate Archiver data for test dataset ckan_cli archiver update-test public-test-dataset # Datasets need to be assigned to an organisation @@ -129,25 +118,19 @@ echo "Assigning test Datasets to Organisation..." echo "Creating non-organisation group:" group_create=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "silly-walks", "title": "Silly walks", "description": "The Ministry of Silly Walks"}' \ - ${CKAN_ACTION_URL}/group_create + api_call '{"name": "silly-walks", "title": "Silly walks", "description": "The Ministry of Silly Walks"}' group_create ) echo ${group_create} echo "Updating group_admin to have admin privileges in the silly-walks group:" group_admin_update=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "silly-walks", "username": "group_admin", "role": "admin"}' \ - ${CKAN_ACTION_URL}/group_member_create + api_call '{"id": "silly-walks", "username": "group_admin", "role": "admin"}' group_member_create ) echo ${group_admin_update} echo "Updating walker to have editor privileges in the silly-walks group:" walker_update=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "silly-walks", "username": "walker", "role": "editor"}' \ - ${CKAN_ACTION_URL}/group_member_create + api_call '{"id": "silly-walks", "username": "walker", "role": "editor"}' group_member_create ) echo ${walker_update} @@ -166,36 +149,26 @@ add_user_if_needed dr_editor "Data Request Editor" dr_editor@localhost echo "Creating ${DR_ORG_TITLE} Organisation:" DR_ORG=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "'"${DR_ORG_NAME}"'", "title": "'"${DR_ORG_TITLE}"'"}' \ - ${CKAN_ACTION_URL}/organization_create + api_call '{"name": "'"${DR_ORG_NAME}"'", "title": "'"${DR_ORG_TITLE}"'"}' organization_create ) DR_ORG_ID=$(echo $DR_ORG | $PYTHON $APP_DIR/bin/extract-id.py) echo "Assigning test users to ${DR_ORG_TITLE} Organisation:" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "'"${DR_ORG_ID}"'", "object": "dr_admin", "object_type": "user", "capacity": "admin"}' \ - ${CKAN_ACTION_URL}/member_create +api_call '{"id": "'"${DR_ORG_ID}"'", "object": "dr_admin", "object_type": "user", "capacity": "admin"}' member_create -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "'"${DR_ORG_ID}"'", "object": "dr_editor", "object_type": "user", "capacity": "editor"}' \ - ${CKAN_ACTION_URL}/member_create +api_call '{"id": "'"${DR_ORG_ID}"'", "object": "dr_editor", "object_type": "user", "capacity": "editor"}' member_create echo "Creating test dataset for data request organisation:" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "data_request_dataset", "title": "Dataset for data requests", "owner_org": "'"${DR_ORG_ID}"'", +api_call '{"name": "data_request_dataset", "title": "Dataset for data requests", "owner_org": "'"${DR_ORG_ID}"'", "update_frequency": "near-realtime", "author_email": "dr_admin@localhost", "version": "1.0", "license_id": "cc-by-4", -"data_driven_application": "NO", "security_classification": "PUBLIC", "notes": "test", "de_identified_data": "NO"}'\ - ${CKAN_ACTION_URL}/package_create +"data_driven_application": "NO", "security_classification": "PUBLIC", "notes": "test", "de_identified_data": "NO"}' package_create echo "Creating test Data Request:" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"title": "Test Request", "description": "This is an example", "organization_id": "'"${TEST_ORG_ID}"'"}' \ - ${CKAN_ACTION_URL}/create_datarequest +api_call '{"title": "Test Request", "description": "This is an example", "organization_id": "'"${TEST_ORG_ID}"'"}' create_datarequest ## # END. @@ -215,27 +188,21 @@ add_user_if_needed report_admin "Reporting Admin" report_admin@localhost echo "Creating ${REPORT_ORG_TITLE} Organisation:" REPORT_ORG=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "'"${REPORT_ORG_NAME}"'", "title": "'"${REPORT_ORG_TITLE}"'"}' \ - ${CKAN_ACTION_URL}/organization_create + api_call '{"name": "'"${REPORT_ORG_NAME}"'", "title": "'"${REPORT_ORG_TITLE}"'"}' organization_create ) REPORT_ORG_ID=$(echo $REPORT_ORG | $PYTHON $APP_DIR/bin/extract-id.py) echo "Assigning admin user to ${REPORT_ORG_TITLE} Organisation:" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_admin", "object_type": "user", "capacity": "admin"}' \ - ${CKAN_ACTION_URL}/member_create +api_call '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_admin", "object_type": "user", "capacity": "admin"}' member_create echo "Creating test Data Request for reporting:" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"title": "Reporting Request", "description": "Data Request for reporting", "organization_id": "'"${REPORT_ORG_ID}"'"}' \ - ${CKAN_ACTION_URL}/create_datarequest +api_call '{"title": "Reporting Request", "description": "Data Request for reporting", "organization_id": "'"${REPORT_ORG_ID}"'"}' create_datarequest ## # END. # -. ${APP_DIR}/bin/deactivate +. "${APP_DIR}"/bin/deactivate diff --git a/bin/docker-compose.sh b/bin/docker-compose.sh index 405bb7c..82f4a05 100755 --- a/bin/docker-compose.sh +++ b/bin/docker-compose.sh @@ -1,13 +1,15 @@ #!/bin/sh +set -x + # Pass commands to Docker Compose v1 or v2 depending on what is present -if (which docker-compose >/dev/null); then - # Docker Compose v1 - docker-compose $* -elif (docker compose ls >/dev/null); then +if (docker compose ls >/dev/null); then # Docker Compose v2 docker compose $* +elif (which docker-compose >/dev/null); then + # Docker Compose v1 + docker-compose $* else # Docker Compose not found exit 1 diff --git a/bin/init-ext.sh b/bin/init-ext.sh index 71e5187..34a5d2b 100755 --- a/bin/init-ext.sh +++ b/bin/init-ext.sh @@ -32,19 +32,19 @@ install_requirements () { done } -. ${APP_DIR}/bin/activate - +. "${APP_DIR}"/bin/activate +pip install "setuptools>=44.1.0,<71" install_requirements . dev-requirements requirements-dev for extension in . `ls -d $SRC_DIR/ckanext-*`; do install_requirements $extension requirements pip-requirements done # force version that declares itself to be incompatible but actually works -pip install click==7.1.2 +pip install click==8.1.7 pip install -e . installed_name=$(grep '^\s*name=' setup.py |sed "s|[^']*'\([-a-zA-Z0-9]*\)'.*|\1|") # Validate that the extension was installed correctly. if ! pip list | grep "$installed_name" > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi -. $APP_DIR/bin/process-config.sh -. ${APP_DIR}/bin/deactivate +. "${APP_DIR}"/bin/process-config.sh +. "${APP_DIR}"/bin/deactivate diff --git a/bin/init.sh b/bin/init.sh index 392fe5e..9cd9505 100755 --- a/bin/init.sh +++ b/bin/init.sh @@ -4,7 +4,7 @@ # set -e -. ${APP_DIR}/bin/activate +. "${APP_DIR}"/bin/activate CLICK_ARGS="--yes" ckan_cli db clean ckan_cli db init ckan_cli datastore set-permissions | psql "postgresql://datastore_write:pass@postgres-datastore/datastore_test" --set ON_ERROR_STOP=1 diff --git a/bin/process-artifacts.sh b/bin/process-artifacts.sh index e20a7e6..b2066b5 100755 --- a/bin/process-artifacts.sh +++ b/bin/process-artifacts.sh @@ -6,8 +6,9 @@ set -e # Create screenshots directory in case it was not created before. This is to # avoid this script to fail when copying artifacts. -ahoy cli "mkdir -p test/screenshots" +ahoy cli "mkdir -p test/screenshots test/junit" # Copy from the app container to the build host for storage. -mkdir -p /tmp/artifacts/behave +mkdir -p /tmp/artifacts/behave /tmp/artifacts/junit docker cp "$(sh bin/docker-compose.sh ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/ +docker cp "$(sh bin/docker-compose.sh ps -q ckan)":/srv/app/test/junit /tmp/artifacts/ diff --git a/bin/serve.sh b/bin/serve.sh index 531b22b..a4a5a4b 100755 --- a/bin/serve.sh +++ b/bin/serve.sh @@ -1,21 +1,7 @@ #!/usr/bin/env sh set -e -dockerize -wait tcp://postgres:5432 -timeout 1m -dockerize -wait tcp://solr:8983 -timeout 1m -dockerize -wait tcp://redis:6379 -timeout 1m - -for i in `seq 1 60`; do - if (PGPASSWORD=pass psql -h postgres -U ckan_default -d ckan_test -c "\q"); then - echo "Database became ready on attempt $i" - break - else - echo "Database not yet ready, retrying (attempt $i)..." - sleep 1 - fi -done - -. ${APP_DIR}/bin/activate +. "${APP_DIR}"/bin/activate if (which ckan > /dev/null); then ckan -c ${CKAN_INI} run --disable-reloader --threaded else diff --git a/ckanext/harvester_data_qld_geoscience/plugin.py b/ckanext/harvester_data_qld_geoscience/plugin.py index e860d5f..fa68181 100644 --- a/ckanext/harvester_data_qld_geoscience/plugin.py +++ b/ckanext/harvester_data_qld_geoscience/plugin.py @@ -11,10 +11,7 @@ from ckan.lib.helpers import json from ckan import model from ckanext.harvest.model import HarvestObject -try: - from collections import OrderedDict # from python 2.7 -except ImportError: - from sqlalchemy.util import OrderedDict +from collections import OrderedDict log = logging.getLogger(__name__) diff --git a/dev-requirements-2.9-py2.txt b/dev-requirements-2.9-py2.txt deleted file mode 100644 index 7dec48a..0000000 --- a/dev-requirements-2.9-py2.txt +++ /dev/null @@ -1,34 +0,0 @@ -behave==1.2.6 -behaving==2.0.0 -Appium-Python-Client<=0.52 -ckanapi==4.3 -ckantoolkit>=0.0.4 -factory-boy -faker==3.0.1 -pytest-factoryboy -flake8==3.8.3 -mock -pytest-ckan -six>=1.13.0 -splinter>=0.13.0,<0.17 -xlrd==1.2.0 -python-magic==0.4.18 -progressbar==2.5 - --e git+https://github.com/ckan/ckanext-dcat.git@v1.2.0#egg=ckanext-dcat --e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming --e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.16#egg=ckanext-archiver --e git+https://github.com/qld-gov-au/ckanext-csrf-filter.git@1.2.1#egg=ckanext-csrf-filter --e git+https://github.com/qld-gov-au/ckanext-data-qld.git@7.3.1#egg=ckanext-data-qld --e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.1-qgov.11#egg=ckanext-datarequests --e git+https://github.com/qld-gov-au/ckanext-harvest.git@v1.4.2.a#egg=ckanext-harvest --e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.10#egg=ckanext-qa --e git+https://github.com/qld-gov-au/ckanext-qgov.git@6.0.2#egg=ckanext-qgov --e git+https://github.com/qld-gov-au/ckanext-report.git@0.3.1-qgov.2#egg=ckanext-report --e git+https://github.com/qld-gov-au/ckanext-resource-type-validation.git@1.0.7#egg=ckanext-resource-type-validation --e git+https://github.com/qld-gov-au/ckanext-ssm-config.git@0.0.2#egg=ckanext-ssm-config --e git+https://github.com/qld-gov-au/ckanext-validation.git@v0.0.8-qgov.11#egg=ckanext-validation --e git+https://github.com/qld-gov-au/ckanext-xloader.git@1.0.1-qgov.4#egg=ckanext-xloader --e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0-qgov.16#egg=ckanext-ytp-comments --e git+https://github.com/qld-gov-au/ckanext-resource-visibility.git@2.1.1#egg=ckanext-resource-visibility --e git+https://github.com/qld-gov-au/ckanext-validation-schema-generator.git@develop#egg=ckanext-validation-schema-generator diff --git a/dev-requirements-2.9.txt b/dev-requirements-2.9.txt index 4e69360..1913d68 100644 --- a/dev-requirements-2.9.txt +++ b/dev-requirements-2.9.txt @@ -5,28 +5,32 @@ ckantoolkit>=0.0.4 factory-boy Faker flake8==3.8.3 +idna>=3.7 # not directly required, pinned by Snyk to avoid a vulnerability mock +progressbar==2.5 pytest-ckan +python-magic==0.4.18 six>=1.13.0 splinter>=0.13.0,<0.17 xlrd==1.2.0 -python-magic==0.4.18 -progressbar==2.5 +requests>=2.32.0 # not directly required, pinned by Snyk to avoid a vulnerability +urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability +zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability --e git+https://github.com/ckan/ckanext-dcat.git@v1.2.0#egg=ckanext-dcat +-e git+https://github.com/ckan/ckanext-dcat.git@v2.1.0#egg=ckanext-dcat -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming --e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.16#egg=ckanext-archiver +-e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.18#egg=ckanext-archiver -e git+https://github.com/qld-gov-au/ckanext-csrf-filter.git@1.2.1#egg=ckanext-csrf-filter -e git+https://github.com/qld-gov-au/ckanext-data-qld.git@7.3.1#egg=ckanext-data-qld --e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.1-qgov.11#egg=ckanext-datarequests +-e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.1-qgov.13#egg=ckanext-datarequests -e git+https://github.com/qld-gov-au/ckanext-harvest.git@v1.4.2.a#egg=ckanext-harvest --e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.10#egg=ckanext-qa --e git+https://github.com/qld-gov-au/ckanext-qgov.git@6.0.2#egg=ckanext-qgov +-e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.12#egg=ckanext-qa +-e git+https://github.com/qld-gov-au/ckanext-qgov.git@7.0.0#egg=ckanext-qgov -e git+https://github.com/qld-gov-au/ckanext-report.git@0.3.1-qgov.2#egg=ckanext-report --e git+https://github.com/qld-gov-au/ckanext-resource-type-validation.git@1.0.7#egg=ckanext-resource-type-validation --e git+https://github.com/qld-gov-au/ckanext-ssm-config.git@0.0.2#egg=ckanext-ssm-config --e git+https://github.com/qld-gov-au/ckanext-validation.git@v0.0.8-qgov.11#egg=ckanext-validation --e git+https://github.com/qld-gov-au/ckanext-xloader.git@1.0.1-qgov.4#egg=ckanext-xloader --e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0-qgov.16#egg=ckanext-ytp-comments +-e git+https://github.com/qld-gov-au/ckanext-resource-type-validation.git@1.0.10#egg=ckanext-resource-type-validation +-e git+https://github.com/qld-gov-au/ckanext-ssm-config.git@0.1.0#egg=ckanext-ssm-config +-e git+https://github.com/qld-gov-au/ckanext-validation.git@v0.0.8-qgov.13#egg=ckanext-validation +-e git+https://github.com/qld-gov-au/ckanext-xloader.git@1.0.1-qgov.16#egg=ckanext-xloader +-e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0#egg=ckanext-ytp-comments -e git+https://github.com/qld-gov-au/ckanext-resource-visibility.git@2.1.1#egg=ckanext-resource-visibility --e git+https://github.com/qld-gov-au/ckanext-validation-schema-generator.git@develop#egg=ckanext-validation-schema-generator +-e git+https://github.com/qld-gov-au/ckanext-validation-schema-generator.git@1.0.4#egg=ckanext-validation-schema-generator diff --git a/dev-requirements.txt b/dev-requirements.txt index f010180..293dd30 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,33 +1,38 @@ behaving==3.1.5 Appium-Python-Client==2.10.1 +certifi>=2024.7.4 # not directly required, pinned by Snyk to avoid a vulnerability ckanapi==4.3 ckantoolkit>=0.0.4 factory-boy Faker flake8==6.0.0 +idna>=3.7 # not directly required, pinned by Snyk to avoid a vulnerability mock +progressbar==2.5 pytest-ckan +python-magic==0.4.18 +requests>=2.32.0 # not directly required, pinned by Snyk to avoid a vulnerability selenium<4.10 six>=1.13.0 typer<0.11 # We still need Click 7 for now +urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability xlrd==1.2.0 -python-magic==0.4.18 -progressbar==2.5 +zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability --e git+https://github.com/ckan/ckanext-dcat.git@v1.2.0#egg=ckanext-dcat +-e git+https://github.com/ckan/ckanext-dcat.git@v2.1.0#egg=ckanext-dcat -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming --e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.16#egg=ckanext-archiver +-e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.18#egg=ckanext-archiver -e git+https://github.com/qld-gov-au/ckanext-csrf-filter.git@1.2.1#egg=ckanext-csrf-filter -e git+https://github.com/qld-gov-au/ckanext-data-qld.git@7.3.1#egg=ckanext-data-qld --e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.1-qgov.11#egg=ckanext-datarequests +-e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.1-qgov.13#egg=ckanext-datarequests -e git+https://github.com/qld-gov-au/ckanext-harvest.git@v1.4.2.a#egg=ckanext-harvest --e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.10#egg=ckanext-qa --e git+https://github.com/qld-gov-au/ckanext-qgov.git@6.0.2#egg=ckanext-qgov +-e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.12#egg=ckanext-qa +-e git+https://github.com/qld-gov-au/ckanext-qgov.git@7.0.0#egg=ckanext-qgov -e git+https://github.com/qld-gov-au/ckanext-report.git@0.3.1-qgov.2#egg=ckanext-report --e git+https://github.com/qld-gov-au/ckanext-resource-type-validation.git@1.0.7#egg=ckanext-resource-type-validation --e git+https://github.com/qld-gov-au/ckanext-ssm-config.git@0.0.2#egg=ckanext-ssm-config --e git+https://github.com/qld-gov-au/ckanext-validation.git@v0.0.8-qgov.11#egg=ckanext-validation --e git+https://github.com/qld-gov-au/ckanext-xloader.git@1.0.1-qgov.4#egg=ckanext-xloader --e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0-qgov.16#egg=ckanext-ytp-comments +-e git+https://github.com/qld-gov-au/ckanext-resource-type-validation.git@1.0.10#egg=ckanext-resource-type-validation +-e git+https://github.com/qld-gov-au/ckanext-ssm-config.git@0.1.0#egg=ckanext-ssm-config +-e git+https://github.com/qld-gov-au/ckanext-validation.git@v0.0.8-qgov.13#egg=ckanext-validation +-e git+https://github.com/qld-gov-au/ckanext-xloader.git@1.0.1-qgov.16#egg=ckanext-xloader +-e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0#egg=ckanext-ytp-comments -e git+https://github.com/qld-gov-au/ckanext-resource-visibility.git@2.1.1#egg=ckanext-resource-visibility --e git+https://github.com/qld-gov-au/ckanext-validation-schema-generator.git@develop#egg=ckanext-validation-schema-generator +-e git+https://github.com/qld-gov-au/ckanext-validation-schema-generator.git@1.0.4#egg=ckanext-validation-schema-generator diff --git a/docker-compose.yml b/docker-compose.yml index ac9e445..ac3402b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,8 @@ services: condition: service_healthy solr: condition: service_started + redis: + condition: service_started networks: - amazeeio-network - default @@ -73,7 +75,7 @@ services: - default solr: - image: ckan/ckan-solr:${CKAN_VERSION} + image: ckan/ckan-solr:${CKAN_VERSION}-solr8 ports: - "8983" environment: diff --git a/test/features/comments.feature b/test/features/comments.feature index fa8b7b4..95670ed 100644 --- a/test/features/comments.feature +++ b/test/features/comments.feature @@ -60,6 +60,16 @@ Feature: Comments When I submit a comment with subject "Test subject" and comment "He had sheep, and oxen, and he asses, and menservants, and maidservants, and she asses, and camels." Then I should see "Comment blocked due to profanity" within 5 seconds + @comment-add @comment-profane + Scenario: When a logged-in user submits a comment containing profanity with special symbols on a dataset they should receive an error message and the comment will not appear + Given "TestOrgEditor" as the persona + When I log in + And I create a dataset with key-value parameters "notes=Profane Dataset Comment with regex characters" + And I go to dataset "$last_generated_name" comments + Then I should see the add comment form + When I submit a comment with subject "Test subject" and comment "Rachel Lindt's cape name is Bi+ch." + Then I should see "Comment blocked due to profanity" within 5 seconds + @comment-add @comment-profane Scenario: When a logged-in user submits a comment containing whitelisted profanity on a Dataset the comment should display within 10 seconds Given "TestOrgEditor" as the persona diff --git a/test/features/data_qld_theme.feature b/test/features/data_qld_theme.feature index bd7f72c..f7d4689 100644 --- a/test/features/data_qld_theme.feature +++ b/test/features/data_qld_theme.feature @@ -104,6 +104,17 @@ Feature: Theme customisations And I should see an element with xpath "//a[contains(@href, '/datastore/dump/') and contains(@href, 'format=json') and contains(string(), 'JSON')]" And I should see an element with xpath "//a[contains(@href, '/datastore/dump/') and contains(@href, 'format=xml') and contains(string(), 'XML')]" + @unauthenticated + @OpenData + Scenario: Open Data - Menu items are present and correct + Given "Unauthenticated" as the persona + When I go to dataset page + Then I should see an element with xpath "//li[contains(@class, 'active')]/a[contains(string(), 'Data') and (@href='/dataset' or @href='/dataset/')]" + And I should see an element with xpath "//li[not(contains(@class, 'active'))]/a[contains(string(), 'Visualisations') and @href='/visualisations']" + And I should see an element with xpath "//li[not(contains(@class, 'active'))]/a[contains(string(), 'News and Case Studies') and @href='/news-and-case-studies']" + And I should see an element with xpath "//li[not(contains(@class, 'active'))]/a[contains(string(), 'Standards and guidance') and @href='/article/standards-and-guidance']" + And I should see an element with xpath "//li[not(contains(@class, 'active'))]/a[contains(string(), 'Contact') and @href='/article/contact']" + @unauthenticated Scenario: When I encounter a 'resource not found' error page, it has a custom message Given "Unauthenticated" as the persona @@ -129,10 +140,3 @@ Feature: Theme customisations When I go to "/robots.txt" Then I should see "Disallow: /" And I should not see "Allow:" - - @unauthenticated - Scenario: When I go to the home page, I can see Visualisations and News and Case Studies in the navbar - Given "Unauthenticated" as the persona - When I go to homepage - Then I should see an element with xpath "//a[string()='Visualisations']" - And I should see an element with xpath "//a[string()='News and Case Studies']" diff --git a/test/features/data_usability_rating.feature b/test/features/data_usability_rating.feature index 438ff13..7d4bf01 100644 --- a/test/features/data_usability_rating.feature +++ b/test/features/data_usability_rating.feature @@ -9,9 +9,9 @@ Feature: Data usability rating And I create a dataset and resource with key-value parameters "license=other-open" and "format=::upload=" And I press the element with xpath "//ol[contains(@class, 'breadcrumb')]//a[starts-with(@href, '/dataset/')]" And I reload page every 3 seconds until I see an element with xpath "//div[contains(@class, 'qa') and contains(@class, 'openness-')]" but not more than 10 times - Then I should see "Data usability rating" + Then I should see data usability rating When I press "Test Resource" - Then I should see an element with xpath "//div[contains(@class, 'qa openness-')]" + Then I should see data usability rating Examples: Formats | Format | Filename | Score | diff --git a/test/features/datasets.feature b/test/features/datasets.feature index 1f9a727..9871ef4 100644 --- a/test/features/datasets.feature +++ b/test/features/datasets.feature @@ -15,9 +15,13 @@ Feature: Dataset APIs Then I should see "created the dataset" When I press "View this version" Then I should see "You're currently viewing an old version of this dataset." - When I go to dataset "$last_generated_name" - And I press the element with xpath "//a[contains(@href, '/dataset/activity/') and contains(string(), 'Activity Stream')]" + + When I go back And I press "Changes" Then I should see "View changes from" And I should see an element with xpath "//select[@name='old_id']" And I should see an element with xpath "//select[@name='new_id']" + + When I go back + And I press the element with xpath "//li[contains(@class, 'new-package')]/preceding-sibling::li[1]//a[contains(string(), 'Changes')]" + Then I should see "Added resource" diff --git a/test/features/organisations.feature b/test/features/organisations.feature index eac7063..f60aa52 100644 --- a/test/features/organisations.feature +++ b/test/features/organisations.feature @@ -37,6 +37,9 @@ Feature: Organization APIs Then I should see "Test Organisation" And I should not see an element with xpath "//a[contains(@href, '?action=read')]" And I should see an element with xpath "//a[contains(@href, '/organization/test-organisation')]" + When I press "Test Organisation" + And I press "Activity Stream" + Then I should see "created the organisation" When I view the "test-organisation" organisation API "not including" users Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "test-organisation"')]" diff --git a/test/features/resource_type_validation.feature b/test/features/resource_type_validation.feature new file mode 100644 index 0000000..30286cf --- /dev/null +++ b/test/features/resource_type_validation.feature @@ -0,0 +1,16 @@ +@resource_type_validation +Feature: Resource type validation + + Scenario: As an evil user, when I try to upload a resource with a MIME type not matching its extension, I should get an error + Given "TestOrgEditor" as the persona + When I log in + And I create a dataset with key-value parameters "notes=Testing resource type mismatch" + And I open the new resource form for dataset "$last_generated_name" + And I create a resource with key-value parameters "name=Testing EICAR PDF::description=Testing EICAR sample virus file with PDF extension::format=PDF::upload=eicar.com.pdf" + Then I should see "Mismatched file type" + + Scenario: As a publisher, when I create a resource linking to an internal URL, I should not see any type mismatch errors + Given "TestOrgEditor" as the persona + When I log in + And I create a dataset and resource with key-value parameters "notes=Testing internal URL" and "name=Internal link::url=http://ckan:5000/api/action/status_show" + Then I should see "Testing internal URL" diff --git a/test/features/search_facets.feature b/test/features/search_facets.feature index 3b00083..17b9a47 100644 --- a/test/features/search_facets.feature +++ b/test/features/search_facets.feature @@ -1,5 +1,7 @@ +@OpenData Feature: Search facets + @unauthenticated Scenario: When I go to the dataset list page, I can see the 'Data Portals' facet Given "Unauthenticated" as the persona When I go to dataset page diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py index 7adb265..54e71a3 100644 --- a/test/features/steps/steps.py +++ b/test/features/steps/steps.py @@ -94,10 +94,11 @@ def log_in_directly(context): :return: """ - assert context.persona, "A persona is required to log in, found [{}] in context. Have you configured the personas in before_scenario?".format(context.persona) + assert context.persona, "A persona is required to log in, found [{}] in context." \ + " Have you configured the personas in before_scenario?".format(context.persona) context.execute_steps(u""" When I attempt to log in with password "$password" - Then I should see an element with xpath "//*[@title='Log out']/i[contains(@class, 'fa-sign-out')]" + Then I should see an element with xpath "//*[@title='Log out' or @data-bs-title='Log out']/i[contains(@class, 'fa-sign-out')]" """) @@ -186,9 +187,13 @@ def go_to_new_resource_form(context, name): """) else: # Existing dataset, browse to the resource form + if context.browser.is_element_present_by_xpath( + "//a[contains(string(), 'Resources') and contains(@href, '/dataset/resources/')]"): + context.execute_steps(u""" + When I press "Resources" + """) context.execute_steps(u""" - When I press "Resources" - And I press "Add new resource" + When I press "Add new resource" And I take a debugging screenshot """) @@ -567,7 +572,7 @@ def create_resource_from_params(context, resource_params): """.format(key, value)) context.execute_steps(u""" When I take a debugging screenshot - And I press the element with xpath "//form[contains(@class, 'resource-form')]//button[contains(@class, 'btn-primary')]" + And I press the element with xpath "//form[contains(@data-module, 'resource-form')]//button[contains(@class, 'btn-primary')]" And I take a debugging screenshot """) @@ -588,11 +593,7 @@ def filter_contents(mail): payload_bytes = quopri.decodestring(payload) if len(payload_bytes) > 0: payload_bytes += b'=' # do fix the padding error issue - if six.PY2: - decoded_payload = payload_bytes.decode('base64') - else: - import base64 - decoded_payload = six.ensure_text(base64.b64decode(six.ensure_binary(payload_bytes))) + decoded_payload = six.ensure_text(base64.b64decode(six.ensure_binary(payload_bytes))) print('Searching for', text, ' and ', text2, ' in decoded_payload: ', decoded_payload) return text in decoded_payload and (not text2 or text2 in decoded_payload) @@ -811,6 +812,16 @@ def go_to_data_request_comments(context, subject): """ % (subject)) +# ckanext-qa + + +@then(u'I should see data usability rating {score}') +def data_usability_rating_visible(context, score): + context.execute_steps(u""" + Then I should see an element with xpath "//div[contains(@class, 'openness-{0}')]" + """.format(score)) + + # ckanext-report diff --git a/test/features/users.feature b/test/features/users.feature index 0d561a2..43f08e1 100644 --- a/test/features/users.feature +++ b/test/features/users.feature @@ -118,6 +118,16 @@ Feature: User APIs Then I should see my datasets And I should see "Add Dataset" + Scenario: Dashboard news feed can display organisational changes + Given "SysAdmin" as the persona + When I log in + And I go to organisation page + And I press "Test Organisation" + And I press "Manage" + And I press "Update" + And I visit "/dashboard" + Then I should see an element with xpath "//li[contains(string(), 'updated the organisation')]/a[contains(string(), 'Test Organisation') and contains(@href, '/organization/')]/..//a[contains(string(), 'Administrator') and @href='/user/admin']" + @email Scenario: As a registered user, when I have locked my account with too many failed logins, I can reset my password to unlock it Given "CKANUser" as the persona @@ -153,3 +163,21 @@ Feature: User APIs And I fill in "password2" with "password1234" And I press "Create Account" Then I should see "Password: Must contain at least one number, lowercase letter, capital letter, and symbol" + + Scenario: As a sysadmin, when I go to the sysadmin list, I can promote and demote other sysadmins + Given "SysAdmin" as the persona + When I log in + And I click the link to a url that contains "/ckan-admin/" + And I take a debugging screenshot + Then I should see an element with xpath "//table//a[string() = 'Administrator' and @href = '/user/admin']" + And I should not see "Test Admin" + + When I fill in "promote-username" with "test_org_admin" + And I press "Promote" + And I take a debugging screenshot + Then I should see "Promoted Test Admin to sysadmin" + And I should see an element with xpath "//table//a[string() = 'Test Admin' and @href = '/user/test_org_admin']" + + When I press the element with xpath "//tr/td/a[@href = '/user/test_org_admin']/../following-sibling::td//button[contains(@title, 'Revoke')]" + Then I should see "Revoked sysadmin permission from Test Admin" + And I should not see an element with xpath "//table//a[@href = '/user/test_org_admin']" diff --git a/test/features/xloader.feature b/test/features/xloader.feature new file mode 100644 index 0000000..a39096c --- /dev/null +++ b/test/features/xloader.feature @@ -0,0 +1,36 @@ +@OpenData +@XLoader +Feature: XLoader + + Scenario: As a publisher, when I visit a resource I control with a datastore entry, I can access the XLoader interface + Given "TestOrgEditor" as the persona + When I log in + And I create a dataset and resource with key-value parameters "notes=Testing XLoader" and "name=test-csv-resource::url=https://people.sc.fsu.edu/~jburkardt/data/csv/addresses.csv::format=CSV" + # Wait for XLoader to run + And I press "test-csv-resource" + And I reload page every 3 seconds until I see an element with xpath "//*[contains(string(), 'DataStore')]" but not more than 6 times + Then I should see "DataStore" + + When I press "DataStore" + And I reload page every 3 seconds until I see an element with xpath "//*[contains(string(), 'Express Load completed')]" but not more than 6 times + Then I should see "Express Load completed" + And I should see "Data Schema" + And I should see "Data Dictionary" + And I should see "Upload to DataStore" + And I should see "Delete from DataStore" + And I should see "Status" + And I should see "Last updated" + And I should see "Upload Log" + And I should see "View resource" + + When I press "Upload to DataStore" + Then I should see "Status" + And I should see "Pending" + And I should see "Delete from DataStore" + + When I press "Delete from DataStore" + And I confirm the dialog containing "delete the DataStore" if present + Then I should see "DataStore and Data Dictionary deleted for resource" + And I should see "Upload to DataStore" + And I should not see "Delete from DataStore" + And I should not see an element with xpath "//a[contains(@href, '/dictionary/')]" diff --git a/test/fixtures/eicar.com.pdf b/test/fixtures/eicar.com.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a2463df6d6421be9fe983071e5a69d379f09c84e GIT binary patch literal 68 zcmazr^;Zl~b#M%bHi-#{iZC_N2#C`(cXrk^uT^pNbao8V4GwYib8!rE(RK6-@eK0} X3Jul`aSaa9b@TLbRaEiN_Rs