From 9a5c7ffa5f952fa695d350848a4bc6d2dffe6613 Mon Sep 17 00:00:00 2001 From: Ihor Kalnytskyi Date: Wed, 1 Nov 2023 23:54:39 +0200 Subject: [PATCH] Use general approach for rendering in commonmark Instead of supporting graphviz and any other possible text-to-diagram providers directly, let's build a general approach instead where a user can specify whatever executable they want to pipe the content of the code block through it, and replace the whole node w/ its output. This drops direct graphviz support in favor of built-in 'exec' pipe. --- .github/workflows/ci.yml | 3 -- pyproject.toml | 5 +-- src/holocron/_processors/commonmark.py | 50 ++++++++++++----------- tests/_processors/test_commonmark.py | 56 ++------------------------ 4 files changed, 32 insertions(+), 82 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb22e38..e5da3c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,8 +34,5 @@ jobs: with: python-version: "3.10" - - name: Set up prerequisites - run: sudo apt-get install graphviz libgraphviz-dev - - name: Test run: pipx run -- hatch run test:run diff --git a/pyproject.toml b/pyproject.toml index 5860620..923bd4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ dependencies = [ "Markdown >= 3.4", "docutils >= 0.19", "feedgen >= 0.9", - "pygraphviz >= 1.10", "termcolor >= 1.1", "colorama >= 0.4", "markdown-it-py >= 2.1", @@ -77,10 +76,10 @@ scripts.fmt = ["ruff --fix {args:.}", "black {args:.}"] [tool.ruff] select = ["F", "E", "W", "I", "S", "FBT", "B", "C4", "DTZ", "T10", "ISC", "RET", "SLF", "RUF"] -ignore = ["S701", "B904"] +ignore = ["S603", "S701", "B904"] [tool.ruff.isort] known-first-party = ["holocron"] [tool.ruff.per-file-ignores] -"tests/*" = ["E501", "S101", "S603", "S607", "SLF001", "RUF001"] +"tests/*" = ["E501", "S101", "S607", "SLF001", "RUF001"] diff --git a/src/holocron/_processors/commonmark.py b/src/holocron/_processors/commonmark.py index 8b0f6a3..cb4b107 100644 --- a/src/holocron/_processors/commonmark.py +++ b/src/holocron/_processors/commonmark.py @@ -2,7 +2,8 @@ import json import logging -import pathlib +import subprocess +import typing as t import markdown_it import markdown_it.renderer @@ -11,13 +12,10 @@ import pygments.formatters.html import pygments.lexers import pygments.util -import pygraphviz from mdit_py_plugins.container import container_plugin from mdit_py_plugins.deflist import deflist_plugin from mdit_py_plugins.footnote import footnote_plugin -import holocron - from ._misc import parameters _LOGGER = logging.getLogger("holocron") @@ -28,22 +26,36 @@ def __init__(self, parser): super().__init__(parser) self._parser = parser - def fence(self, tokens, idx, options, env): + def fence(self, tokens, idx, options, env) -> str: token = tokens[idx] + match token.info.split(maxsplit=1): - case ["dot", params]: - env.setdefault("diagrams", []) + case [_, params]: params = json.loads(params) - diagram_name = f"diagram-{len(env['diagrams'])}.svg" - diagram_data = pygraphviz.AGraph(token.content).draw( - format=params["format"], - prog=params.get("engine", "dot"), - ) - env["diagrams"].append((diagram_name, diagram_data)) - return self._parser.render(f"![]({diagram_name})") + if "exec" in params: + standard_input = token.content.encode("UTF-8") + standard_output = _exec_pipe(params["exec"], standard_input) + return standard_output.decode("UTF-8") + return super().fence(tokens, idx, options, env) +def _exec_pipe(args: t.List[str], input_: t.ByteString, timeout: int = 1000) -> bytes: + try: + completed_process = subprocess.run( + args, + input=input_, + capture_output=True, + timeout=timeout, + check=True, + ) + except subprocess.TimeoutExpired: + return b"timed out executing the command" + except subprocess.CalledProcessError as exc: + return exc.stderr + return completed_process.stdout + + def _pygmentize(code: str, language: str, _: str) -> str: if not language: return code @@ -118,13 +130,3 @@ def process( item["content"] = commonmark.renderer.render(tokens, commonmark.options, env) item["destination"] = item["destination"].with_suffix(".html") yield item - - for diagram_name, diagram_bytes in env.get("diagrams", []): - yield holocron.WebSiteItem( - { - "source": pathlib.Path("dot://", str(item["source"]), diagram_name), - "destination": item["destination"].with_name(diagram_name), - "baseurl": app.metadata["url"], - "content": diagram_bytes, - } - ) diff --git a/tests/_processors/test_commonmark.py b/tests/_processors/test_commonmark.py index 26cdf3e..db73d8e 100644 --- a/tests/_processors/test_commonmark.py +++ b/tests/_processors/test_commonmark.py @@ -407,7 +407,7 @@ def test_args_pygmentize_unknown_language(testapp, language): ] -def test_item_dot_render(testapp): +def test_item_exec(testapp): """Commonmark has to render DOT snippets into SVG if asked to render.""" stream = commonmark.process( @@ -417,10 +417,8 @@ def test_item_dot_render(testapp): { "content": textwrap.dedent( """ - ```dot {"format": "svg"} - graph yoda { - a -- b -- c - } + ```text {"exec": ["sed", "--expression", "s/ a / the /g"]} + yoda, a jedi grandmaster ``` """ ), @@ -435,57 +433,11 @@ def test_item_dot_render(testapp): assert list(stream) == [ holocron.Item( { - "content": '

\n', + "content": "yoda, the jedi grandmaster\n", "source": pathlib.Path("1.md"), "destination": pathlib.Path("1.html"), }, ), - holocron.WebSiteItem( - { - "content": _pytest_regex(rb".*\s*", re.DOTALL | re.MULTILINE), - "source": pathlib.Path("dot://1.md/diagram-0.svg"), - "destination": pathlib.Path("diagram-0.svg"), - "baseurl": "https://yoda.ua", - } - ), - ] - - -def test_item_dot_not_rendered(testapp): - """Commonmark has to preserve DOT snippet if not asked to render.""" - - stream = commonmark.process( - testapp, - [ - holocron.Item( - { - "content": textwrap.dedent( - """ - ```dot - graph yoda { - a -- b -- c - } - ``` - """ - ), - "destination": pathlib.Path("1.md"), - } - ) - ], - ) - - assert isinstance(stream, collections.abc.Iterable) - assert list(stream) == [ - holocron.Item( - { - "content": ( - '
graph yoda {\n'
-                    "    a -- b -- c\n"
-                    "}\n
\n" - ), - "destination": pathlib.Path("1.html"), - } - ) ]