[QOLSVC-8587] sync tests with Open Data theme

.ahoy.yml

@@ -144,22 +144,20 @@ commands:
       ahoy cli "rm -f test/screenshots/*"
       ahoy start-ckan-job-workers
       ahoy start-mailmock &
-      sleep 5 &&
+      sleep 5
       if [ "$BEHAVE_TAG" = "" ]; then
         # no tag specified, probably running locally
-        # Run reporting tests last so numbers are consistent
         (ahoy cli "behave -k ${*:-test/features} --tags=smoke" \
-         && ahoy cli "behave -k ${*:-test/features} --tags=-smoke --tags=-reporting" \
-         && ahoy cli "behave -k ${*:-test/features} --tags=reporting" \
+         && ahoy cli "behave -k ${*:-test/features} --tags=-smoke" \
         ) || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       elif [ "$BEHAVE_TAG" = "authenticated" ]; then
         # run any tests that don't have a specific tag
-        ahoy cli "behave -k ${*:-test/features} --tags=-unauthenticated --tags=-smoke --tags=-reporting" \
+        ahoy cli "behave -k ${*:-test/features} --tags=-unauthenticated --tags=-smoke --tags=-OpenData --tags=-multi_plugin" \
           || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
-        # Run reporting tests last so numbers are consistent
-        ahoy cli "behave -k ${*:-test/features} --tags=reporting" || \
-          [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       else
+        if [ "$BEHAVE_TAG" != "multi_plugin" ]; then
+            BEHAVE_TAG="$BEHAVE_TAG --tags=-multi_plugin"
+        fi
         # run tests with the specified tag
         ahoy cli "behave -k ${*:-test/features} --tags=$BEHAVE_TAG" \
           || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]

.docker/test.ini

@@ -56,6 +56,7 @@ ckan.datastore.default_fts_index_method = gist
 
 ## Site Settings.
 ckan.site_url = http://ckan:5000/
+WTF_CSRF_ENABLED = False
 
 ## Authorization Settings
 

.github/workflows/test.yml

@@ -12,8 +12,8 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
       - name: Install requirements
@@ -26,18 +26,22 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        ckan-version: ["2.10", 2.9, 2.9-py2]
-        behave-tag: [smoke, authenticated, unauthenticated]
+        ckan-version: ["2.10", 2.9]
+        behave-tag: [smoke, unauthenticated, multi_plugin, OpenData, authenticated]
 
     name: Run ${{ matrix.behave-tag }} tests on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
-    container: drevops/ci-builder:23.7.0
+    container: drevops/ci-runner:23.12.0
     env:
       CKAN_VERSION: ${{ matrix.ckan-version }}
       BEHAVE_TAG: ${{ matrix.behave-tag }}
 
     steps:
-      - uses: actions/checkout@v3
+      # Patch https://github.com/actions/runner/issues/863
+      - name: Preserve $HOME set in the container
+        run: echo HOME=/root >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v4
         timeout-minutes: 2
 
       - name: Build
@@ -65,7 +69,7 @@ jobs:
 
       - name: Upload screenshots
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: CKAN ${{ matrix.ckan-version }} ${{ matrix.behave-tag }} screenshots
           path: /tmp/artifacts/behave/screenshots

bin/build.sh

@@ -20,7 +20,7 @@ CKAN_GIT_VERSION=$CKAN_VERSION
 CKAN_GIT_ORG=qld-gov-au
 
 if [ "$CKAN_VERSION" = "2.10" ]; then
-    CKAN_GIT_VERSION=ckan-2.10.1-qgov.4
+    CKAN_GIT_VERSION=ckan-2.10.5-qgov.4
     PYTHON_VERSION=py3
     PYTHON="${PYTHON}3"
 else

bin/create-test-data.sh

@@ -118,10 +118,11 @@ curl -LsH "Authorization: ${API_KEY}" \
 "license_id": "other-open", "data_driven_application": "NO", "security_classification": "PUBLIC",
 "notes": "public test", "de_identified_data": "NO", "resources": [
     {"name": "test-resource", "description": "Test resource description",
-     "url": "https://example.com", "format": "HTML", "size": 1024}
+     "url": "https://example.com/foo", "format": "HTML", "size": 1024}
 ]}' \
     ${CKAN_ACTION_URL}/package_create
 
+ckan_cli archiver update-test public-test-dataset
 # Datasets need to be assigned to an organisation
 
 echo "Assigning test Datasets to Organisation..."

dev-requirements-2.9-py2.txt

@@ -17,18 +17,18 @@ progressbar==2.5
 
 -e git+https://github.com/ckan/ckanext-dcat.git@v1.2.0#egg=ckanext-dcat
 -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming
--e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.15#egg=ckanext-archiver
--e git+https://github.com/qld-gov-au/ckanext-csrf-filter.git@1.1.8#egg=ckanext-csrf-filter
--e git+https://github.com/qld-gov-au/ckanext-data-qld.git@7.2.3#egg=ckanext-data-qld
--e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.1-qgov.9#egg=ckanext-datarequests
+-e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.16#egg=ckanext-archiver
+-e git+https://github.com/qld-gov-au/ckanext-csrf-filter.git@1.2.1#egg=ckanext-csrf-filter
+-e git+https://github.com/qld-gov-au/ckanext-data-qld.git@7.3.1#egg=ckanext-data-qld
+-e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.1-qgov.11#egg=ckanext-datarequests
 -e git+https://github.com/qld-gov-au/ckanext-harvest.git@v1.4.2.a#egg=ckanext-harvest
--e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.9#egg=ckanext-qa
+-e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.10#egg=ckanext-qa
 -e git+https://github.com/qld-gov-au/ckanext-qgov.git@6.0.2#egg=ckanext-qgov
 -e git+https://github.com/qld-gov-au/ckanext-report.git@0.3.1-qgov.2#egg=ckanext-report
--e git+https://github.com/qld-gov-au/ckanext-resource-type-validation.git@1.0.5#egg=ckanext-resource-type-validation
+-e git+https://github.com/qld-gov-au/ckanext-resource-type-validation.git@1.0.7#egg=ckanext-resource-type-validation
 -e git+https://github.com/qld-gov-au/ckanext-ssm-config.git@0.0.2#egg=ckanext-ssm-config
 -e git+https://github.com/qld-gov-au/ckanext-validation.git@v0.0.8-qgov.11#egg=ckanext-validation
 -e git+https://github.com/qld-gov-au/ckanext-xloader.git@1.0.1-qgov.4#egg=ckanext-xloader
--e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0-qgov.15#egg=ckanext-ytp-comments
+-e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0-qgov.16#egg=ckanext-ytp-comments
 -e git+https://github.com/qld-gov-au/ckanext-resource-visibility.git@2.1.1#egg=ckanext-resource-visibility
 -e git+https://github.com/qld-gov-au/ckanext-validation-schema-generator.git@develop#egg=ckanext-validation-schema-generator

dev-requirements-2.9.txt

@@ -15,18 +15,18 @@ progressbar==2.5
 
 -e git+https://github.com/ckan/ckanext-dcat.git@v1.2.0#egg=ckanext-dcat
 -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming
--e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.15#egg=ckanext-archiver
--e git+https://github.com/qld-gov-au/ckanext-csrf-filter.git@1.1.8#egg=ckanext-csrf-filter
--e git+https://github.com/qld-gov-au/ckanext-data-qld.git@7.2.3#egg=ckanext-data-qld
--e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.1-qgov.9#egg=ckanext-datarequests
+-e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.16#egg=ckanext-archiver
+-e git+https://github.com/qld-gov-au/ckanext-csrf-filter.git@1.2.1#egg=ckanext-csrf-filter
+-e git+https://github.com/qld-gov-au/ckanext-data-qld.git@7.3.1#egg=ckanext-data-qld
+-e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.1-qgov.11#egg=ckanext-datarequests
 -e git+https://github.com/qld-gov-au/ckanext-harvest.git@v1.4.2.a#egg=ckanext-harvest
--e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.9#egg=ckanext-qa
+-e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.10#egg=ckanext-qa
 -e git+https://github.com/qld-gov-au/ckanext-qgov.git@6.0.2#egg=ckanext-qgov
 -e git+https://github.com/qld-gov-au/ckanext-report.git@0.3.1-qgov.2#egg=ckanext-report
--e git+https://github.com/qld-gov-au/ckanext-resource-type-validation.git@1.0.5#egg=ckanext-resource-type-validation
+-e git+https://github.com/qld-gov-au/ckanext-resource-type-validation.git@1.0.7#egg=ckanext-resource-type-validation
 -e git+https://github.com/qld-gov-au/ckanext-ssm-config.git@0.0.2#egg=ckanext-ssm-config
 -e git+https://github.com/qld-gov-au/ckanext-validation.git@v0.0.8-qgov.11#egg=ckanext-validation
 -e git+https://github.com/qld-gov-au/ckanext-xloader.git@1.0.1-qgov.4#egg=ckanext-xloader
--e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0-qgov.15#egg=ckanext-ytp-comments
+-e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0-qgov.16#egg=ckanext-ytp-comments
 -e git+https://github.com/qld-gov-au/ckanext-resource-visibility.git@2.1.1#egg=ckanext-resource-visibility
 -e git+https://github.com/qld-gov-au/ckanext-validation-schema-generator.git@develop#egg=ckanext-validation-schema-generator

dev-requirements.txt

@@ -15,18 +15,18 @@ progressbar==2.5
 
 -e git+https://github.com/ckan/ckanext-dcat.git@v1.2.0#egg=ckanext-dcat
 -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming
--e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.15#egg=ckanext-archiver
--e git+https://github.com/qld-gov-au/ckanext-csrf-filter.git@1.1.8#egg=ckanext-csrf-filter
--e git+https://github.com/qld-gov-au/ckanext-data-qld.git@7.2.3#egg=ckanext-data-qld
--e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.1-qgov.9#egg=ckanext-datarequests
+-e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.16#egg=ckanext-archiver
+-e git+https://github.com/qld-gov-au/ckanext-csrf-filter.git@1.2.1#egg=ckanext-csrf-filter
+-e git+https://github.com/qld-gov-au/ckanext-data-qld.git@7.3.1#egg=ckanext-data-qld
+-e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.1-qgov.11#egg=ckanext-datarequests
 -e git+https://github.com/qld-gov-au/ckanext-harvest.git@v1.4.2.a#egg=ckanext-harvest
--e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.9#egg=ckanext-qa
+-e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.10#egg=ckanext-qa
 -e git+https://github.com/qld-gov-au/ckanext-qgov.git@6.0.2#egg=ckanext-qgov
 -e git+https://github.com/qld-gov-au/ckanext-report.git@0.3.1-qgov.2#egg=ckanext-report
--e git+https://github.com/qld-gov-au/ckanext-resource-type-validation.git@1.0.5#egg=ckanext-resource-type-validation
+-e git+https://github.com/qld-gov-au/ckanext-resource-type-validation.git@1.0.7#egg=ckanext-resource-type-validation
 -e git+https://github.com/qld-gov-au/ckanext-ssm-config.git@0.0.2#egg=ckanext-ssm-config
 -e git+https://github.com/qld-gov-au/ckanext-validation.git@v0.0.8-qgov.11#egg=ckanext-validation
 -e git+https://github.com/qld-gov-au/ckanext-xloader.git@1.0.1-qgov.4#egg=ckanext-xloader
--e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0-qgov.15#egg=ckanext-ytp-comments
+-e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0-qgov.16#egg=ckanext-ytp-comments
 -e git+https://github.com/qld-gov-au/ckanext-resource-visibility.git@2.1.1#egg=ckanext-resource-visibility
 -e git+https://github.com/qld-gov-au/ckanext-validation-schema-generator.git@develop#egg=ckanext-validation-schema-generator

test/features/admin_reporting.feature

@@ -1,5 +1,6 @@
 @reporting
 @OpenData
+@multi_plugin
 Feature: AdminReporting
 
     Scenario: As an admin user of my organisation, I can view 'My Reports' tab in the dashboard and show the 'Admin Report' with filters and table columns

test/features/comments.feature

@@ -1,5 +1,6 @@
 @comments
 @OpenData
+@multi_plugin
 Feature: Comments
 
     @unauthenticated
@@ -30,7 +31,7 @@ Feature: Comments
 
     @comment-add @datarequest
     Scenario: When a logged-in user submits a comment on a Data Request the comment should then be visible on the Comments tab of the Data Request
-        Given "CKANUser" as the persona
+        Given "TestOrgEditor" as the persona
         When I log in
         And I create a datarequest
         And I go to data request "$last_generated_title" comments
@@ -40,7 +41,7 @@ Feature: Comments
 
     @comment-add @datarequest @email
     Scenario: When a logged-in user submits a comment on a Data Request the email should contain title and comment
-        Given "CKANUser" as the persona
+        Given "TestOrgEditor" as the persona
         When I log in
         And I create a datarequest
         And I go to data request "$last_generated_title" comments
@@ -71,7 +72,7 @@ Feature: Comments
 
     @comment-add @comment-profane @datarequest
     Scenario: When a logged-in user submits a comment containing profanity on a Data Request they should receive an error message and the comment will not appear
-        Given "CKANUser" as the persona
+        Given "TestOrgEditor" as the persona
         When I log in
         And I create a datarequest
         And I go to data request "$last_generated_title" comments
@@ -96,7 +97,7 @@ Feature: Comments
 
     @comment-report @datarequest @email
     Scenario: When a logged-in user reports a comment on a Data Request the comment should be marked as reported and an email notification sent to the organisation admins
-        Given "CKANUser" as the persona
+        Given "TestOrgEditor" as the persona
         When I log in
         And I create a datarequest
         And I go to data request "$last_generated_title" comments

test/features/data_qld_theme.feature

@@ -129,3 +129,10 @@ Feature: Theme customisations
         When I go to "/robots.txt"
         Then I should see "Disallow: /"
         And I should not see "Allow:"
+
+    @unauthenticated
+    Scenario: When I go to the home page, I can see Visualisations and News and Case Studies in the navbar
+        Given "Unauthenticated" as the persona
+        When I go to homepage
+        Then I should see an element with xpath "//a[string()='Visualisations']"
+        And I should see an element with xpath "//a[string()='News and Case Studies']"

test/features/data_usability_rating.feature

@@ -1,13 +1,14 @@
 @data_usability_rating
 @OpenData
+@multi_plugin
 Feature: Data usability rating
 
     Scenario Outline: As a publisher, when I create a resource with an open license, I can verify the openness score is correct
         Given "TestOrgEditor" as the persona
         When I log in
         And I create a dataset and resource with key-value parameters "license=other-open" and "format=<Format>::upload=<Filename>"
         And I press the element with xpath "//ol[contains(@class, 'breadcrumb')]//a[starts-with(@href, '/dataset/')]"
-        And I reload page every 3 seconds until I see an element with xpath "//div[contains(@class, 'qa') and contains(@class, 'openness-')]" but not more than 3 times
+        And I reload page every 3 seconds until I see an element with xpath "//div[contains(@class, 'qa') and contains(@class, 'openness-')]" but not more than 10 times
         Then I should see "Data usability rating"
         When I press "Test Resource"
         Then I should see an element with xpath "//div[contains(@class, 'qa openness-<Score>')]"
@@ -20,3 +21,15 @@ Feature: Data usability rating
             | CSV    | csv_resource.csv   | 3     |
             | JSON   | json_resource.json | 3     |
             | RDF    | rdf_resource.rdf   | 4     |
+
+    Scenario: As a publisher, when I create an open resource with a matching schema, I can verify the score is upgraded from 3 to 4
+        Given "TestOrgEditor" as the persona
+        When I log in
+        And I create a dataset and resource with key-value parameters "license=other-open" and "format=CSV::upload=test_game_data.csv::schema=default"
+        And I press the element with xpath "//ol[contains(@class, 'breadcrumb')]//a[starts-with(@href, '/dataset/')]"
+        And I reload page every 3 seconds until I see an element with xpath "//div[contains(@class, 'qa') and contains(@class, 'openness-4')]" but not more than 10 times
+        Then I should not see an element with xpath "//div[contains(@class, 'openness-3')]"
+        And I should see an element with xpath "//div[contains(@class, 'openness-4')]"
+        When I take a debugging screenshot
+        And I press "Test Resource"
+        Then I should see an element with xpath "//div[contains(@class, 'qa openness-4')]"

test/features/data_validation.feature

@@ -1,5 +1,6 @@
 @config
 @OpenData
+@multi_plugin
 Feature: Data Validation
 
     Scenario Outline: As a sysadmin, admin and editor user of the dataset organisation I can see the 'JSON' button

test/features/datarequest.feature

@@ -1,5 +1,6 @@
 @datarequest
 @OpenData
+@multi_plugin
 Feature: Data Request
 
     @unauthenticated
@@ -54,6 +55,16 @@ Feature: Data Request
         And I should see an element with the css selector "span.error-block" within 1 seconds
         And I should see "Description cannot be empty" within 1 seconds
 
+    Scenario: When a logged-in user submits a Data Request containing profanity they should receive an error message and the request will not be created
+        Given "TestOrgEditor" as the persona
+        When I log in
+        And I go to the data requests page
+        And I press "Add data request"
+        And I fill in title with random text
+        And I fill in "description" with "He had sheep, and oxen, and he asses, and menservants, and maidservants, and she asses, and camels."
+        And I press the element with xpath "//button[contains(@class, 'btn-primary') and contains(string(), 'Create data request')]"
+        Then I should see "Blocked due to profanity" within 5 seconds
+
     Scenario: Data request creator and Sysadmin can see a 'Close' button on the data request detail page for opened data requests
         Given "SysAdmin" as the persona
         When I log in
@@ -92,6 +103,51 @@ Feature: Data Request
         Then I should see an element with xpath "//i[contains(@class, 'icon-lock')]"
         And I should not see an element with xpath "//a[contains(string(), 'Close')]"
 
+    Scenario: Delete all data requests for a user
+        # Use a different profile so we don't delete any other test requests
+        Given "TestOrgAdmin" as the persona
+        When I log in
+        And I create a datarequest
+        And I create a datarequest
+        And I create a datarequest
+        And I go to the "test_org_admin" profile page
+        And I take a debugging screenshot
+        And I press the element with xpath "//ul[contains(@class, 'nav-tabs')]//a[contains(string(), 'Data Requests')]"
+        And I take a debugging screenshot
+        Then I should see an element with xpath "//a[@title='Delete' and contains(@class, 'btn-danger')]"
+        And I should not see an element with xpath "//a[contains(@class, 'btn-danger-serious')]"
+
+        Given "SysAdmin" as the persona
+        When I log out
+        And I log in
+        And I go to the "test_org_admin" profile page
+        And I take a debugging screenshot
+        And I press the element with xpath "//ul[contains(@class, 'nav-tabs')]//a[contains(string(), 'Data Requests')]"
+        And I take a debugging screenshot
+        Then I should see an element with xpath "//a[@title='Delete' and contains(@class, 'btn-danger')]"
+        And I should see an element with xpath "//a[contains(@title, 'Delete all') and contains(@class, 'btn-danger-serious')]"
+
+        When I press the element with xpath "//a[contains(@title, 'Delete all') and contains(@class, 'btn-danger-serious')][1]"
+        And I take a debugging screenshot
+        And I confirm the dialog containing "Are you sure you want to" if present
+        And I take a debugging screenshot
+        Then I should see an element with xpath "//div[contains(@class, 'alert') and contains(string(), 'Deleted') and contains(string(), 'data request(s)')]"
+
+        When I go to the "test_org_admin" profile page
+        And I press the element with xpath "//ul[contains(@class, 'nav-tabs')]//a[contains(string(), 'Data Requests')]"
+        Then I should see "No data requests found"
+
+    Scenario: An unprivileged user who tries to create multiple data requests close together should see an error
+        Given "CKANUser" as the persona
+        When I log in
+        And I create a datarequest
+        And I go to the data requests page
+        And I press "Add data request"
+        And I fill in title with random text
+        And I fill in "description" with "Test throttling"
+        And I press the element with xpath "//button[contains(@class, 'btn-primary')]"
+        Then I should see "Too many requests submitted, please wait"
+
     Scenario: As an org admin I can re-open a closed data request
         Given "DataRequestOrgAdmin" as the persona
         When I log in