refactor testing: extend plugin tests, get rid of scripts (#305)

* WIP

* module test work

* roles *should* run?

* use devel orb for now

* docker-compose => docker compose

* fix "not a tty errors"

* try limiting to cgroupv1

* final fixes

* what the fuck

* use 3 executors for module tests

* lower further

* aa

* cleanup molecule config and galaxy.yml

* update documentation

.circleci/config.yml

@@ -1,31 +1,7 @@
 version: 2.1
 
 orbs:
-  collection-testing: maxhoesel-ansible/ansible-collection-testing@0.3.8
-
-jobs:
-  modules-sanity:
-    executor: collection-testing/default
-    steps:
-      - checkout
-      - collection-testing/install-podman-on-ubuntu
-      - collection-testing/install-requirements-txt
-      - run:
-          command: tests/test-modules-sanity
-          environment:
-            TEST_RETRIES: 3
-            TEST_RETRY_DELAY: 300
-  modules-integration:
-    executor: collection-testing/default
-    steps:
-      - checkout
-      - collection-testing/install-podman-on-ubuntu
-      - collection-testing/install-requirements-txt
-      - run:
-          command: tests/test-modules-integration
-          environment:
-            TEST_RETRIES: 3
-            TEST_RETRY_DELAY: 300
+  collection-testing: maxhoesel-ansible/ansible-collection-testing@0.4.0
 
 filters: &semver-tagged
   tags:
@@ -37,18 +13,30 @@ filters: &semver-tagged
 workflows:
   ci:
     jobs:
-      - collection-testing/pre-commit-lint
-      - collection-testing/antsibull-docs
-      - collection-testing/tox-role-scenarios:
-          parallelism: 16
+      - collection-testing/pre-commit-lint:
+          name: Lint
+      - collection-testing/antsibull-docs:
+          name: Generate Docs
+      - collection-testing/run-tox-environments:
+          name: Test Modules
+          match-environments: \-test\-
+          parallelism: 3
+          resource-class: medium
+          retries: 1
+          retry-delay: 60
+      - collection-testing/run-tox-environments:
+          name: Test Roles
+          match-environments: roles
+          # number of scenarios * ansible versions to test
+          parallelism: 12
           resource-class: large
-          retries: 3
-          retry-delay: 300
+          retries: 1
+          retry-delay: 60
       - collection-testing/publish-github:
+          name: Publish Release to GitHub
           context: collection-publishing
           filters: *semver-tagged
       - collection-testing/publish-galaxy:
+          name: Publish to Galaxy
           context: collection-publishing
           filters: *semver-tagged
-      - modules-sanity
-      - modules-integration

.ansible-lint → .config/ansible-lint.yaml

@@ -1,4 +1,5 @@
-# Don't automatically install roles from galaxy to prevent flaky tests
+---
+# Don't automatically install roles from galaxy
 offline: true
 
 exclude_paths:

.config/molecule/config.yml

@@ -0,0 +1,36 @@
+---
+dependency:
+  name: galaxy
+
+driver:
+  name: podman
+
+provisioner:
+  name: ansible
+  env: {}
+    # Cannot enable pipelining for podman at this time:
+    # https://github.com/ansible-community/molecule-podman/issues/2
+    #ANSIBLE_PIPELINING: false
+  inventory:
+    group_vars:
+      all:
+        # Versions to use, passed in from Tox
+        step_cli_version: ${STEP_CLI_VERSION}
+        step_ca_version: ${STEP_CA_VERSION}
+
+scenario:
+  test_sequence:
+    - destroy
+    - dependency
+    - syntax
+    - create
+    - prepare
+    - converge
+    - idempotence
+    - check  # also run check mode in regular tests
+    - side_effect
+    - verify
+    - destroy
+
+verifier:
+  name: ansible

.readthedocs.yml

@@ -5,7 +5,7 @@ python:
     - requirements: docs/requirements.txt
 
 build:
-  os: "ubuntu-20.04"
+  os: "ubuntu-22.04"
   tools:
     python: "3.10"
   apt_packages:

README.md

@@ -12,20 +12,6 @@ and the [CLI tool](https://github.com/smallstep/cli). Possible uses for this col
 - Token or certificate creation from within your Ansible playbooks
 - [Complete configuration of client certificates via ACME, including automatic renewal](roles/step_acme_cert/README.md)
 
----
-**⚠️ SUPPORTED SMALLSTEP VERSION NOTICE ⚠️**
-
-The smallstep tools (and this collection) are constantly changing and breaking changes may be introduced in each minor version (e.g. from `0.20` to `0.21`).
-To help maintain compatibility, you should use the version of this collection that corresponds to your `step-cli` version.
-
-For example, if you are using `step-cli==0.20`, you should use the collection version `>=0.20,<0.21`.
-Newer and older collection versions *may* work, but are not supported.
-
-**For step-cli versions `<0.20`:** Use the collection version `>=0.4,<0.5`.
-This was the last collection version released under the old versioning scheme.
-
----
-
 ## Components
 
 ---
@@ -47,15 +33,6 @@ This was the last collection version released under the old versioning scheme.
 
 ### Modules
 
-#### CA Modules
-
----
-**NOTE**
-
-To learn more about the differences between Online/Offline/Local-Only Modules, see [this section](#module-usage)
-
----
-
 | Module  | Description | Remote (Online mode) | Local (Offline mode) |
 |---------|-------------|--------|---------------|
 | [`step_ca_bootstrap`](https://ansible-collection-smallstep.readthedocs.io/en/latest/collections/maxhoesel/smallstep/step_ca_bootstrap_module.html) | Initialize `step-cli` to trust a step-ca server | ✅ | ❌ |
@@ -65,34 +42,34 @@ To learn more about the differences between Online/Offline/Local-Only Modules, s
 | [`step_ca_revoke`](https://ansible-collection-smallstep.readthedocs.io/en/latest/collections/maxhoesel/smallstep/step_ca_revoke_module.html) | Revoke a Certificate | ✅ | `offline` parameter |
 | [`step_ca_token`](https://ansible-collection-smallstep.readthedocs.io/en/latest/collections/maxhoesel/smallstep/step_ca_token_module.html) | Generate an OTT granting access to the CA | ✅ | `offline` parameter |
 
-#### Standalone Modules
-
-None so far
-
 ## Installation
 
 ### Dependencies
 
-- A recent release of Ansible. This collection officially supports the 3 most recent Ansible releases.
+- A recent release of Ansible. This collection is tested against the 3 most recent Ansible releases.
   Older versions might still work, but are not supported
-- Python 3.6 or newer on the target host
+- Python 3.6 or newer on the target nodes
 
 Individual roles or modules may have additional dependencies, please check their respective documentation.
 
+### Versioning Policy and Node Requirements
+
+Each minor version of this collection designed to be compatible with the corresponding minor release of the `step-cli` utility.
+For example, The collection releases with version `0.24.x` are compatible with the `step-cli` utility versions `0.24.x`.
+This coupling is needed as newer minor versions of the `step-cli` tool may introduce breaking changes and affect this collection.
+
+To install the correct collection version, check your `step-cli` version (`step-cli --version`), then use that value when installing the collection.
+
+**For step-cli versions `<0.20`:** Use the collection version `>=0.4,<0.5`.
+
 ### Install
 
 Via ansible-galaxy (recommended):
 
 `ansible-galaxy collection install maxhoesel.smallstep>=your-step-cli-version,<next-major-version`
 
-For more information about supported step-cli versions, see the support notice at the top of this file.
-
 Alternatively, you can download a collection archive from a [previous release](hhttps://github.com/maxhoesel-ansible/ansible-collection-smallstep/releases).
 
-You can also clone this repository directly if you want a slightly more up-to-date (and potentially buggy) version.
-
-`ansible-galaxy collection install git+https://github.com/maxhoesel-ansible/ansible-collection-smallstep`
-
 ## Module Usage
 
 This collection contains several modules for managing your smallstep environment.

galaxy.yml

@@ -3,34 +3,34 @@ authors:
 build_ignore:
 - .circleci
 - .github
+- .config
+- .vscode
+- dist
 - CODE_OF_CONDUCT.md
 - CONTRIBUTING.md
-- pyproject.toml
-- tox.ini
 - .yamllint
-- .ansible-lint
-- .pre-commit-config.yaml
 - '*.tar.gz'
 - .venv
 - .tox
 - .pytest_cache
 - scripts
+- .pre-commit-config.yaml
+- .readthedocs.yaml
+- pyproject.toml
 - requirements.txt
-- requirements-molecule.txt
-- '**/requirements-molecule.txt'
+- tox.ini
 - '**/requirements.txt'
 dependencies: {}
-description: Mangage Smallstep CA servers
+description: Install, configure and use the Smallstep CA server and CLI tool
 issues: https://github.com/maxhoesel-ansible/ansible-collection-smallstep/issues
-license:
-- GPL-3.0-or-later
-license_file: ''
+license_file: LICENSE
 name: smallstep
 namespace: maxhoesel
 readme: README.md
 repository: https://github.com/maxhoesel-ansible/ansible-collection-smallstep
 tags:
 - smallstep
 - ca
+- application
 - certificates
 version: 0.24.1

requirements.txt

@@ -1,15 +1,20 @@
-# Module deps, useful for editor intellisense, etc.
-ansible-core==2.15.1
+# Requirements for developing this collectio
+# Includes utilities, CLI helpers and so on
 
 # Linting & Formatting
 ansible-lint==6.17.2
 pylint==2.17.4
 autopep8==2.0.2
 pre-commit==3.3.3
 
-# Testing
+# Test scenario runner
 tox==3.28.0
 
-# Utility packages needed for scripts
+# Utility packages used in scripts
 pyyaml==6.0.0
 packaging==23.1
+
+# Also include a version of ansible-core for IDE hints and such.
+# Note that this version is independent of the one used in tests, see tox.ini
+# and /tests
+ansible-core==2.15.1

roles/step_acme_cert/molecule/default/molecule.yml

@@ -1,14 +1,9 @@
-driver:
-  name: podman
 platforms:
-  # We use the images provided by geerlingguy where possible, as they provide out-of-the-box
-  # support for Ansible (pre_build_image=true, speeds up testing).
-
-  # Use the smallstep-provided CA image for testing
+  # Use the smallstep-provided CA image so that we don't have to set up the CA ourselves
   - name: step-ca
     groups:
       - ca
-    image: "docker.io/smallstep/step-ca:${STEP_CA_VERSION:-latest}"
+    image: "docker.io/smallstep/step-ca:${STEP_CA_VERSION}"
     # we don't actually use the container with ansible, leave it as is
     override_command: false
     pre_build_image: true
@@ -98,37 +93,11 @@ platforms:
     network: molecule-step-acme-cert
 
 provisioner:
-  name: ansible
-  env:
-    # This is required for podman to function: https://github.com/ansible-community/molecule-podman/issues/2
-    ANSIBLE_PIPELINING: false
-    #ANSIBLE_VERBOSITY: 3 # enable for debugging
   inventory:
     group_vars:
       ca:
         step_ca_user: step-ca
       all:
-        step_cli_version: ${STEP_CLI_VERSION:-latest}
-        step_ca_version: ${STEP_CA_VERSION:-latest}
         step_cli_steppath: /etc/step-cli-molecule
-
         step_acme_cert_ca_provisioner: ACME
         step_bootstrap_ca_url: https://step-ca:9000
-
-scenario:
-  test_sequence:
-    - lint
-    - destroy
-    - dependency
-    - syntax
-    - create
-    - prepare
-    - converge
-    - idempotence
-    - check # also run check mode in regular tests
-    - side_effect
-    - verify
-    - destroy
-
-verifier:
-  name: ansible

roles/step_acme_cert/molecule/default/requirements.txt

@@ -1 +1 @@
-../../../../requirements-molecule.txt
+../../../../tests/roles/requirements.txt