🔧 Improve spec update script (#361)

Author: chrisjsewell

.github/workflows/benchmark.yml

@@ -14,7 +14,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: 3.10
+        python-version: "3.10"
 
     - name: install pandoc
       uses: r-lib/actions/setup-pandoc@v2

.github/workflows/tests.yml

@@ -45,6 +45,9 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -e .[testing,linkify]
+    - name: Check spec file is up to date
+      run: |
+        python tests/test_cmark_spec/get_cmark_spec.py
     - name: Run pytest
       run: |
         pytest tests/ --cov=markdown_it --cov-report=xml --cov-report=term-missing

docs/contributing.md

@@ -61,6 +61,22 @@ For documentation build tests:
 >> make html-strict
 ```
 
+### Updating the commonmark specification
+
+If you need to update the commonmark specification, you can do so by running:
+
+```shell
+>> cd markdown-it-py
+>> python tests/test_cmark_spec/get_cmark_spec.py
+```
+
+or
+
+```shell
+>> cd markdown-it-py
+>> uv run tests/test_cmark_spec/get_cmark_spec.py
+```
+
 ## Contributing a plugin
 
 1. Does it already exist as JavaScript implementation ([see npm](https://www.npmjs.com/search?q=keywords:markdown-it-plugin))?

pyproject.toml

@@ -60,6 +60,7 @@ testing = [
     "pytest",
     "pytest-cov",
     "pytest-regressions",
+    "requests",
 ]
 benchmarking = [
     "psutil",

tests/test_cmark_spec/get_cmark_spec.py

@@ -0,0 +1,77 @@
+# /// script
+# dependencies = [
+#   "requests",
+# ]
+# ///
+from pathlib import Path
+
+default_version = "0.30"
+default_output_path = Path(__file__).parent / "commonmark.json"
+default_fixture_test_path = (
+    Path(__file__).parent.parent / "test_port" / "fixtures" / "commonmark_spec.md"
+)
+
+
+def create_argparser():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Download CommonMark spec JSON")
+    parser.add_argument(
+        "version",
+        nargs="?",
+        default=default_version,
+        help=f"CommonMark spec version to download (default: {default_version})",
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=Path,
+        default=default_output_path,
+        help=f"Output file path (default: {default_output_path})",
+    )
+    parser.add_argument(
+        "--test-fixture",
+        type=Path,
+        default=default_fixture_test_path,
+        help=f"Write to test fixture (default: {default_fixture_test_path})",
+    )
+    return parser
+
+
+if __name__ == "__main__":
+    import requests  # type: ignore[import-untyped]
+
+    args = create_argparser().parse_args()
+    version: str = args.version
+    output_path: Path = args.output
+    write_to_test_fixture = True
+    test_fixture: Path = args.test_fixture
+    changed = False
+    url = f"https://spec.commonmark.org/{version}/spec.json"
+    print(f"Downloading CommonMark spec from {url}")
+    response = requests.get(url)
+    response.raise_for_status()
+    if not output_path.exists() or output_path.read_text() != response.text:
+        changed = True
+        with output_path.open("w") as f:
+            f.write(response.text)
+        print(f"Updated to {output_path}")
+    else:
+        print(f"Spec file {output_path} is up to date, not overwriting")
+
+    if write_to_test_fixture:
+        data = response.json()
+        text = ""
+        for item in data:
+            text += "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
+            text += f"src line: {item['start_line'] - 1}\n\n"
+            text += f".\n{item['markdown']}.\n{item['html']}.\n\n"
+        if not test_fixture.exists() or test_fixture.read_text() != text:
+            changed = True
+            with test_fixture.open("w") as f:
+                f.write(text)
+            print(f"Also updated to {test_fixture}")
+        else:
+            print(f"Fixture file {test_fixture} is up to date, not overwriting")
+
+    raise SystemExit(0 if not changed else 1)