Dependencies: Make installing plugin packages optional

.github/workflows/cd.yml

@@ -35,15 +35,15 @@ jobs:
     - name: Install Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.9'
+        python-version: '3.11'
         cache: pip
         cache-dependency-path: pyproject.toml
 
     - name: Update pip and install wheel
       run: pip install -U pip wheel
 
     - name: Install Python package and dependencies
-      run: pip install -e .[docs,pre-commit,tests]
+      run: pip install -e .[all_plugins,docs,pre-commit,tests]
 
     - name: Run pre-commit
       run: pre-commit run --all-files || ( git status --short ; git diff ; exit 1 )
@@ -78,15 +78,41 @@ jobs:
       run: pip install -U pip wheel
 
     - name: Install Python package and dependencies
+      run: pip install -e .[all_plugins,tests]
+
+    - name: Run pytest
+      run: pytest -sv -m "not minimal_install" tests
+
+  tests-minimal-install:
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    strategy:
+      matrix:
+        python-version: ['3.9']
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: pip
+        cache-dependency-path: pyproject.toml
+
+    - name: Install Python package without any extras
       run: pip install -e .[tests]
 
     - name: Run pytest
-      run: pytest -sv tests
+      run: pytest -sv tests -m minimal_install
+
 
   publish:
 
     name: Publish to PyPI
-    needs: [pre-commit, tests]
+    needs: [pre-commit, tests, tests-minimal-install]
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/ci.yml

@@ -14,15 +14,15 @@ jobs:
     - name: Install Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.9'
+        python-version: '3.11'
         cache: pip
         cache-dependency-path: pyproject.toml
 
     - name: Update pip and install wheel
       run: pip install -U pip wheel
 
     - name: Install Python package and dependencies
-      run: pip install -e .[docs,pre-commit,tests]
+      run: pip install -e .[all_plugins,docs,pre-commit,tests]
 
     - name: Run pre-commit
       run: pre-commit run --all-files || ( git status --short ; git diff ; exit 1 )
@@ -56,9 +56,34 @@ jobs:
       run: pip install -U pip wheel
 
     - name: Install Python package and dependencies
-      run: pip install -e .[tests]
+      run: pip install -e .[all_plugins,tests]
 
     - name: Run pytest
       env:
         AIIDA_WARN_v3: true
-      run: pytest -sv tests
+      run: pytest -sv -m "not minimal_install" tests
+
+  tests-minimal-install:
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    strategy:
+      matrix:
+        python-version: ['3.9']
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: pip
+        cache-dependency-path: pyproject.toml
+
+    - name: Install Python package without any extras
+      run: pip install -e .[tests]
+
+    - name: Run pytest
+      run: pytest -sv tests -m minimal_install

.pre-commit-config.yaml

@@ -31,3 +31,11 @@ repos:
     args: [--autofix]
   - id: pretty-format-yaml
     args: [--autofix]
+
+- repo: local
+  hooks:
+  - id: optional-dependencies
+    name: validate optional dependencies
+    entry: python ./dev/validate_optional_dependencies.py
+    language: system
+    files: pyproject.toml|

.readthedocs.yml

@@ -12,6 +12,7 @@ python:
   - method: pip
     path: .
     extra_requirements:
+    - all_plugins
     - docs
 
 sphinx:

dev/validate_optional_dependencies.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+"""Script to validate the optional dependencies in the `pyproject.toml`."""
+
+
+def main():
+    """Validate the optional dependencies."""
+    import pathlib
+    import sys
+
+    import tomllib
+
+    filepath_pyproject_toml = pathlib.Path(__file__).parent.parent / 'pyproject.toml'
+
+    with filepath_pyproject_toml.open('rb') as handle:
+        pyproject = tomllib.load(handle)
+
+    exclude = ['all_plugins', 'docs', 'pre-commit', 'tests']
+    dependencies_all_plugins = pyproject['project']['optional-dependencies']['all_plugins']
+    dependencies_separate = []
+
+    for key, dependencies in pyproject['project']['optional-dependencies'].items():
+        if key in exclude:
+            continue
+        dependencies_separate.extend(dependencies)
+
+    missing_all_plugins = set(dependencies_separate).difference(set(dependencies_all_plugins))
+    excess_all_plugins = set(dependencies_all_plugins).difference(set(dependencies_separate))
+
+    if missing_all_plugins:
+        print(
+            'ERROR: the `all_plugins` extras are inconsistent. The following plugin dependencies are missing: '
+            f'{", ".join(missing_all_plugins)}',
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    if excess_all_plugins:
+        print(
+            'ERROR: the `all_plugins` extras are inconsistent. The following dependencies are not declared by any '
+            f'plugin extras: {", ".join(excess_all_plugins)}',
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

docs/source/index.rst

@@ -62,6 +62,49 @@ The common workflows can be subdivided into two categories:
 
 
 
+.. _installation:
+
+************
+Installation
+************
+
+The Python package can be installed from the Python Package index (PyPI) or directly from the source:
+The recommended method of installation is to use the Python package manager ``pip``:
+
+.. code-block:: console
+
+    $ pip install aiida-common-workflows
+
+This will install the latest stable version that was released to PyPI.
+Note that this will not install any of the plugin packages that are required to run any of the common workflow implementations.
+To install all plugin packages that implement a common workflow, run the install with the ``all_plugins`` extra:
+
+.. code-block:: console
+
+    $ pip install aiida-common-workflows[all_plugins]
+
+Alternatively, you can choose a specific plugin to prevent having to install all plugin packages, for example:
+
+.. code-block:: console
+
+    $ pip install aiida-common-workflows[quantum_espresso]
+
+will install the package plus the dependencies that are required to run the implementation for Quantum ESPRESSO.
+The available extras are ``abinit``, ``bigdft``, ``castep``, ``cp2k``, ``fleur``, ``gaussian``, ``gpaw``, ``nwchem``, ``orca``, ``quantum_espresso``, ``siesta``, ``vasp`` and ``wien2k``.
+
+To install the package from source, first clone the repository and then install using ``pip``:
+
+.. code-block:: console
+
+    $ git clone https://github.com/aiidateam/aiida-common-workflows
+    $ pip install -e aiida-common-workflows
+
+The ``-e`` flag will install the package in editable mode, meaning that changes to the source code will be automatically picked up.
+
+To work with ``aiida-common-workflows``, a configured AiiDA profile is required.
+Please refer to `AiiDA's documentation <https://aiida.readthedocs.io/projects/aiida-core/en/latest/intro/get_started.html>`_ for detailed instructions.
+
+
 .. _how-to-submit:
 
 *******************************
@@ -94,7 +137,7 @@ If more flexibility is required, it is advised to write a custom launch script,
 .. code:: python
 
     from aiida.engine import submit
-    from aiida.plugins import WorkflowFactory
+    from aiida_common_workflows.plugins import WorkflowFactory
 
     RelaxWorkChain = WorkflowFactory('common_workflows.relax.quantum_espresso')  # Load the relax workflow implementation of choice.
 

docs/source/workflows/base/relax/index.rst

@@ -36,7 +36,7 @@ A typical script for the submission of common relax workflow could look somethin
 .. code:: python
 
     from aiida.engine import submit
-    from aiida.plugins import WorkflowFactory
+    from aiida_common_workflows.plugins import WorkflowFactory
 
     RelaxWorkChain = WorkflowFactory('common_workflows.relax.<implementation>')  # Load the relax workflow implementation of choice.
 

docs/source/workflows/composite/dc.rst

@@ -17,7 +17,7 @@ A typical script for the submission of common DC workflow could look something l
 
     from aiida.orm import List, Dict
     from aiida.engine import submit
-    from aiida.plugins import WorkflowFactory
+    from aiida_common_workflows.plugins import WorkflowFactory
 
     cls = WorkflowFactory('common_workflows.dissociation_curve')
 

docs/source/workflows/composite/eos.rst

@@ -15,7 +15,7 @@ A typical script for the submission of common EoS workflow could look something
 
     from aiida.orm import List, Dict
     from aiida.engine import submit
-    from aiida.plugins import WorkflowFactory
+    from aiida_common_workflows.plugins import WorkflowFactory
 
     cls = WorkflowFactory('common_workflows.eos')
 

pyproject.toml

@@ -17,25 +17,9 @@ classifiers = [
   'Programming Language :: Python :: 3.12'
 ]
 dependencies = [
-  'abipy==0.9.6',
-  'aiida-ase~=3.0',
-  'aiida-abinit~=0.5.0',
-  'aiida-bigdft~=0.3.0',
-  'aiida-castep~=2.0',
   'aiida-core[atomic_tools]~=2.1',
-  'aiida-cp2k~=2.0',
-  'aiida-fleur~=2.0',
-  'aiida-gaussian~=2.0',
-  'aiida-nwchem~=3.0',
-  'aiida-orca~=0.6.0',
-  'aiida-pseudo~=1.0',
-  'aiida-quantumespresso~=4.4',
-  'aiida-siesta~=2.0',
-  'aiida-vasp~=3.1',
-  'aiida-wien2k~=0.2.0',
   'ase!=3.20.*',
   'click~=8.0',
-  'masci-tools~=0.9',
   'pint~=0.16',
   'pymatgen>=2022.1.20'
 ]
@@ -65,21 +49,79 @@ requires-python = '>=3.9'
 'common_workflows.relax.wien2k' = 'aiida_common_workflows.workflows.relax.wien2k.workchain:Wien2kCommonRelaxWorkChain'
 
 [project.optional-dependencies]
+abinit = [
+  'abipy==0.9.6',
+  'aiida-abinit~=0.5.0'
+]
+all_plugins = [
+  'abipy==0.9.6',
+  'aiida-abinit~=0.5.0',
+  'aiida-ase~=3.0',
+  'aiida-bigdft~=0.3.0',
+  'aiida-castep~=2.0',
+  'aiida-cp2k~=2.0',
+  'aiida-fleur~=2.0',
+  'aiida-gaussian~=2.0',
+  'aiida-nwchem~=3.0',
+  'aiida-orca~=0.6.0',
+  'aiida-quantumespresso~=4.4',
+  'aiida-siesta~=2.0',
+  'aiida-vasp~=3.1',
+  'aiida-wien2k~=0.2.0',
+  'masci-tools~=0.9'
+]
+bigdft = [
+  'aiida-bigdft~=0.3.0'
+]
+castep = [
+  'aiida-castep~=2.0'
+]
+cp2k = [
+  'aiida-cp2k~=2.0'
+]
 docs = [
   'pydata-sphinx-theme~=0.14.3',
   'sphinx~=7.2',
   'sphinx-copybutton~=0.5.0',
   'sphinx-design~=0.5.0',
   'sphinxcontrib-details-directive~=0.1.0'
 ]
+fleur = [
+  'aiida-fleur~=2.0',
+  'masci-tools~=0.9'
+]
+gaussian = [
+  'aiida-gaussian~=2.0'
+]
+gpaw = [
+  'aiida-ase~=3.0'
+]
+nwchem = [
+  'aiida-nwchem~=3.0'
+]
+orca = [
+  'aiida-orca~=0.6.0'
+]
 pre-commit = [
   'pre-commit~=3.6'
 ]
+quantum_espresso = [
+  'aiida-quantumespresso~=4.4'
+]
+siesta = [
+  'aiida-siesta~=2.0'
+]
 tests = [
   'pytest~=7.2',
   'pgtest~=1.3,>=1.3.1',
   'pytest-regressions~=1.0'
 ]
+vasp = [
+  'aiida-vasp~=3.1'
+]
+wien2k = [
+  'aiida-wien2k~=0.2.0'
+]
 
 [project.scripts]
 acwf = 'aiida_common_workflows.cli:cmd_root'
@@ -120,6 +162,9 @@ filterwarnings = [
   'ignore:Creating AiiDA configuration folder.*:UserWarning',
   'ignore:Object of type .* not in session, .* operation along .* will not proceed:sqlalchemy.exc.SAWarning'
 ]
+markers = [
+  'minimal_install: mark test as relevant for minimal install.'
+]
 testpaths = [
   'tests'
 ]

src/aiida_common_workflows/plugins/__init__.py

@@ -1,4 +1,10 @@
 """Module with utilities for working with the plugins provided by this plugin package."""
 from .entry_point import get_entry_point_name_from_class, get_workflow_entry_point_names, load_workflow_entry_point
+from .factories import WorkflowFactory
 
-__all__ = ('get_workflow_entry_point_names', 'get_entry_point_name_from_class', 'load_workflow_entry_point')
+__all__ = (
+    'WorkflowFactory',
+    'get_workflow_entry_point_names',
+    'get_entry_point_name_from_class',
+    'load_workflow_entry_point',
+)