Feat/10 update and auto deploy sphinx (#27)

* Updated dev-dependencies, added first version of Sphinx documentation, including built html documentation.

* Added Logo, updated theme, updated GitHub workflow, fixed docstrings in cache and cube. Hosting on ReadTheDocs has to be done by Owner/ CorrelAid (but can be requested and triggered that way).

* Updated urllib3 version, but everything <2.0.0 (deprecating `strict`) should be fine...

* Updated poetry as recommended in cachecontrol issue report.

* Fixed black formatting, fixed make docs (is now ran by poetry).

* Fixed linting issue, updated packages, updated make docs.

* Updated ReadMe, added developer sphinx documentation, added custom pre-commit hook and changed to hard-coded version in docs, added built documentation to artifacts, #3

* Add deployment workflow, needs Repo updates

* Update depencies for Sphinx documentation #10

* Remove redundant docu information #10

Render parts of the README.md in the respective .rst files

* Remove unused mdinclude, fix run-test py version, update pre-commit #10

* Fix dependency group for SPhinx workflow #10

* Fix docstring parameter rendering in Sphinx #10

* Fix image rendering by mimicking folder structure #10

* Add comment on warnings related to ext.napoleon #10

* Rename deploy-docs #10

* Fix black format issue in conf.py #10

* Update deploy key, add deploy trigger comment #10

* Update documentation deploy workflow #10

* Switch to matrix.os definition #10

* Fix pull_request target in deploy workflow #10

* Update poetry.lock #10

* Import package version to Sphinx docu #10

* Manually fix black formatting issue #10

* With auto-deploy working, decrease retention days #10

* Update readme and Sphinx header references #10

* Fix deploy to update files on the remote #10

.github/workflows/deploy-docs.yaml

@@ -0,0 +1,61 @@
+name: Deploy docs
+
+on:
+  push:
+    branches:
+      - main
+      - dev
+  pull_request:
+    branches:
+      - main
+      - dev
+  workflow_dispatch:
+
+env:
+  POETRY_VERSION: 1.6.1
+
+jobs:
+  build-and-deploy:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+        os: [ubuntu-latest]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Run poetry image
+        uses: abatilo/actions-poetry@v2.0.0
+        with:
+            poetry-version: $POETRY_VERSION
+      - name: Install dependencies
+        run: |
+          poetry install --with dev
+      - name: Build docs
+        run: |
+          cd docs
+          poetry run make clean
+          poetry run make html
+      # create distinguished subdirectories for each branch
+      # NOTE: force_orphan will delete all files in the branch (thus main & dev could not be concurrently deployed)
+      - name: Deploy docs main
+        uses: peaceiris/actions-gh-pages@v3
+        if: github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.base_ref == 'main')
+        with:
+          deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
+          publish_dir: ./docs/build/html
+          publish_branch: gh-pages
+      - name: Deploy docs dev
+        uses: peaceiris/actions-gh-pages@v3
+        if: github.ref == 'refs/heads/dev' || (github.event_name == 'pull_request' && github.base_ref == 'dev')
+        with:
+          deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
+          publish_dir: ./docs/build/html
+          destination_dir: dev
+          publish_branch: gh-pages

.github/workflows/run-tests.yaml

@@ -44,13 +44,15 @@ jobs:
         run: |
           poetry run pytest --cov=pystatis tests
   code-quality:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
     strategy:
       fail-fast: false
       matrix:
         # only support specific python version, as guidelines differ beween (minor) versions
         python-version: ["3.11.6"]
         os: [ubuntu-latest]
-    runs-on: ${{ matrix.os }}
+
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v2
@@ -76,3 +78,36 @@ jobs:
         run: poetry run safety check
       - name: Run mypy
         run: poetry run mypy src
+  sphinx-documentation:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Run poetry image
+        uses: abatilo/actions-poetry@v2.0.0
+        with:
+            poetry-version: $POETRY_VERSION
+      - name: Install dependencies
+        run: |
+          poetry install --with dev
+      - name: Build documentation
+        run: |
+          cd docs
+          poetry run make clean
+          poetry run make html
+      - name: Upload documentation as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: docs
+          path: docs/build/html/*
+          retention-days: 5

.pre-commit-config.yaml

@@ -29,3 +29,11 @@ repos:
         files: ^\.github/workflows/
         types: [yaml]
         args: ["--schemafile", "https://json.schemastore.org/github-workflow"]
+  - repo: https://github.com/thclark/pre-commit-sphinx
+    rev: 0.0.1
+    hooks:
+      - id: build-docs
+        name: "Check if documentation compiles"
+        args: ['--cache-dir', 'docs/build/doctrees', '--html-dir', 'docs/build/html', '--source-dir', 'docs/source']
+        language_version: python3
+        additional_dependencies: [myst-parser]

README.md

@@ -119,6 +119,18 @@ clear_cache("21311-0001")  # only deletes the data for the object with the speci
 clear_cache()  # deletes the complete cache
 ```
 
+### Full documentation
+
+The full documentation of the main and dev branches are hosted via [GitHub Pages (main)](https://correlaid.github.io/pystatis/) and [GitHub Pages (dev)](https://correlaid.github.io/pystatis/dev/). It can also be built locally by running
+
+```bash
+cd docs && make clean && make html
+```
+
+from the project root directory. Besides providing parsed docstrings of the individual package modules, the full documentation currently mirrors most of the readme, like installation and usage. The mirroring crucially relies on the names of the section headers in the ReadMe, so change them with care!
+
+More information on how to use sphinx is provided [here](https://docs.readthedocs.io/en/stable/intro/getting-started-with-sphinx.html).
+
 ## License
 
 Distributed under the MIT License. See `LICENSE.txt` for more information.
@@ -128,7 +140,6 @@ Distributed under the MIT License. See `LICENSE.txt` for more information.
 A few ideas we should implement in the maybe-near future:
 
 - Improve Table parsing. Right now, the parsing is really simple and we should align the cube and table format so that the data frame for tables is more convenient to use.
-- Create a source code documentation with Sphinx or similar tools.
 - Mechanism to download data that is newer than the cached version. Right now, once data is cached, it is always retrieved from cache no matter if there is a newer version online. However, this could be quite challenging as the GENESIS API is really bad in providing a good and consistent field for the last update datetime.
 - Improve Table and Cube metadata so the user can look up the variables contained in the dataset and for each variable the values that this variable can have.
 - Understand and support time series.

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/source/conf.py

@@ -0,0 +1,55 @@
+import subprocess
+
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "pystatis"
+copyright = "2022, Michael Aydinbas"
+authors = [
+    "Michael Aydinbas <michael.aydinbas@gmail.com>",
+    "Ariz Weber <ariz.weber@protonmail.com>",
+    "CorrelAid <info@correlaid.org>",
+    "Daniel Pleus <danielpleus@gmail.com>",
+    "Felix Schmitz <felix.schmitz@philosophy-economics.de>",
+    "Frederik Hering <jobs.fhering@gmail.com>",
+    "Marco Hübner <marco_huebner1@gmx.de>",
+]
+maintainers = ["Michael Aydinbas <michael.aydinbas@gmail.com>"]
+release = (
+    subprocess.check_output(["poetry", "version"], text=True)
+    .strip()
+    .split()[-1]
+)
+version = release
+
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    "myst_parser",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",  # used to generate overview tables
+    "sphinx.ext.napoleon",  # used for google-style docstrings
+]
+
+templates_path = ["_templates"]
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "sphinx_rtd_theme"
+html_static_path = ["_static"]
+
+html_title = "pystatis"
+html_short_title = "pystatis documentation"
+html_logo = "_static/pystatis_logo.png"
+html_favicon = "_static/pystatis_logo.ico"
+autodoc_typehints = "description"

docs/source/contribute.rst

@@ -0,0 +1,6 @@
+Developer Information
+=====================
+
+.. include:: ../../README.md
+    :parser: myst_parser.sphinx_
+    :start-after: ## How to contribute?

docs/source/index.rst

@@ -0,0 +1,36 @@
+.. pystatis documentation master file, created by
+   sphinx-quickstart on Sun May  7 19:10:45 2023.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+pystatis
+========
+
+.. include:: ../../README.md
+   :parser: myst_parser.sphinx_
+   :start-after: # ``pystatis``
+   :end-before: ## Installation
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents
+
+   install
+   start
+   use
+   roadmap
+   contribute
+   license
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Modules
+
+   pystatis
+
+Indices and tables
+------------------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/source/install.rst

@@ -0,0 +1,7 @@
+Installation
+============
+
+.. include:: ../../README.md
+    :parser: myst_parser.sphinx_
+    :start-after: ## Installation
+    :end-before: ## Getting started

docs/source/license.rst

@@ -0,0 +1,7 @@
+License
+==========
+
+.. include:: ../../README.md
+    :parser: myst_parser.sphinx_
+    :start-after: ## License
+    :end-before: ## Roadmap