test: Use current env for pre-commit tests

* Use current test env to run pre-commit tests

* This avoids creating a new env and installing jupytext in that env

Signed-off-by: Mahendra Paipuri <[email protected]>

Author: mahendrapaipuri

tests/conftest.py

@@ -2,6 +2,7 @@
 import os.path
 import re
 import sys
+import yaml
 import unittest.mock as mock
 from pathlib import Path
 
@@ -62,6 +63,34 @@ def jupytext_repo_rev(jupytext_repo_root):
     """The local revision of this repo, to use in .pre-commit-config.yaml in tests"""
     return system("git", "rev-parse", "HEAD", cwd=jupytext_repo_root).strip()
 
+@pytest.fixture
+def jupytext_pre_commit_config(jupytext_repo_root):
+    """The local revision of this repo, to use in .pre-commit-config.yaml in tests"""
+    # Read pre-commit-hooks.yaml file
+    #
+    # Setting language to system will ensure that pre-commit assumes that we 
+    # provision correct environment. This is the case when we run unit tests as we
+    # make a developmental install of jupytext before running unit tests. So there is
+    # an "test" environment that is provisioned and we use this current environment
+    # to run pre-commit tests.
+    #
+    # When there are additional_dependencies, we override this in individual test to
+    # python so that pre-commit will create an environment and install those 
+    # additional dependencies in that environment.
+    #
+    # This strategy will enable us to directly test the pre-commit hook config with 
+    # current version of jupytext. It also avoid re-installing jupytext in pre-commit
+    # config and thus tests run faster.
+    with open(os.path.join(jupytext_repo_root, ".pre-commit-hooks.yaml"), 'r') as file:
+        pre_commit_hooks = yaml.safe_load(file)
+    pre_commit_hooks[0]["language"] = "system"
+    pre_commit_config = {"repos": [
+        {
+            "repo": "local",
+            "hooks": pre_commit_hooks,
+        }
+    ]}
+    return pre_commit_config
 
 @pytest.fixture()
 def python_notebook():

tests/external/pre_commit/test_pre_commit_0_ipynb_to_py.py

@@ -1,3 +1,5 @@
+import os
+import yaml
 import pytest
 from git.exc import HookExecutionError
 from nbformat.v4.nbbase import new_markdown_cell, new_notebook
@@ -9,25 +11,23 @@
 
 
 def test_pre_commit_hook_ipynb_to_py(
-    tmpdir, cwd_tmpdir, tmp_repo, jupytext_repo_root, jupytext_repo_rev
+    tmpdir, cwd_tmpdir, tmp_repo, jupytext_repo_root, jupytext_repo_rev, jupytext_pre_commit_config
 ):
     """Here we document and test the expected behavior of the pre-commit hook in the
     directional (--to) mode. Note that here, the ipynb file is always the source for
     updates - i.e. changes on the .py file will not trigger the hook.
     """
     # set up the tmpdir repo with pre-commit
-    pre_commit_config_yaml = f"""
-repos:
-- repo: {jupytext_repo_root}
-  rev: {jupytext_repo_rev}
-  hooks:
-  - id: jupytext
-    args: [--from, ipynb, --to, "py:percent"]
-"""
+    # Add args as if we will add in actual .pre-commit-config.yaml file
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = [
+        "--from", "ipynb", "--to", "py:percent"
+    ]
+    with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), 'w') as file:
+        yaml.dump(jupytext_pre_commit_config, file)
 
-    tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
+#     tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
     tmp_repo.git.add(".pre-commit-config.yaml")
-    pre_commit(["install", "--install-hooks"])
+    pre_commit(["install", "--install-hooks", "-f"])
 
     # write test notebook and output file
     nb = new_notebook(cells=[new_markdown_cell("A short notebook")])

tests/external/pre_commit/test_pre_commit_1_sync.py

@@ -1,4 +1,6 @@
+import os
 import shutil
+import yaml
 
 import pytest
 from git.exc import HookExecutionError
@@ -15,17 +17,13 @@ def test_pre_commit_hook_sync(
     tmp_repo,
     jupytext_repo_root,
     jupytext_repo_rev,
+    jupytext_pre_commit_config,
     python_notebook,
 ):
-    pre_commit_config_yaml = f"""
-repos:
-- repo: {jupytext_repo_root}
-  rev: {jupytext_repo_rev}
-  hooks:
-  - id: jupytext
-    args: [--sync]
-"""
-    tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
+    # Add args as if we will add in actual .pre-commit-config.yaml file
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = ["--sync"]
+    with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), 'w') as file:
+        yaml.dump(jupytext_pre_commit_config, file)
 
     tmp_repo.git.add(".pre-commit-config.yaml")
     pre_commit(["install", "--install-hooks", "-f"])

tests/external/pre_commit/test_pre_commit_1_sync_with_config.py

@@ -1,3 +1,5 @@
+import os
+import yaml
 import pytest
 from git.exc import HookExecutionError
 from nbformat.v4.nbbase import new_markdown_cell
@@ -12,17 +14,12 @@ def test_pre_commit_hook_sync_with_config(
     tmp_repo,
     jupytext_repo_root,
     jupytext_repo_rev,
+    jupytext_pre_commit_config,
     python_notebook,
 ):
-    pre_commit_config_yaml = f"""
-repos:
-- repo: {jupytext_repo_root}
-  rev: {jupytext_repo_rev}
-  hooks:
-  - id: jupytext
-    args: [--sync]
-"""
-    tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = ["--sync"]
+    with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), 'w') as file:
+        yaml.dump(jupytext_pre_commit_config, file)
 
     tmp_repo.git.add(".pre-commit-config.yaml")
     pre_commit(["install", "--install-hooks", "-f"])

tests/external/pre_commit/test_pre_commit_1_sync_with_no_config.py

@@ -1,3 +1,5 @@
+import os
+import yaml
 from copy import deepcopy
 
 import pytest
@@ -16,23 +18,14 @@ def test_pre_commit_hook_sync_with_no_config(
     tmp_repo,
     jupytext_repo_root,
     jupytext_repo_rev,
+    jupytext_pre_commit_config,
     notebook_with_outputs,
 ):
     """In this test we reproduce the setting from https://github.com/mwouts/jupytext/issues/967"""
-    pre_commit_config_yaml = f"""
-repos:
-- repo: {jupytext_repo_root}
-  rev: {jupytext_repo_rev}
-  hooks:
-  - id: jupytext
-    args: [
-      '--sync',
-      '--set-formats',
-      'ipynb,py:percent',
-      '--show-changes',
-      '--'
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = [
+        "--sync", "--set-formats", "ipynb,py:percent", "--show-changes", "--"
     ]
-"""
+    
     # Save a sample notebook with outputs in Jupyter
     nb = deepcopy(notebook_with_outputs)
     (tmpdir / "notebooks").mkdir()
@@ -45,7 +38,8 @@ def test_pre_commit_hook_sync_with_no_config(
     tmp_repo.index.commit("Notebook with outputs")
 
     # Configure the pre-commit hook
-    tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
+    with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), 'w') as file:
+        yaml.dump(jupytext_pre_commit_config, file)
     tmp_repo.git.add(".pre-commit-config.yaml")
     pre_commit(["install", "--install-hooks", "-f"])
 

tests/external/pre_commit/test_pre_commit_2_sync_nbstripout.py

@@ -1,3 +1,5 @@
+import os
+import yaml
 import pytest
 from git.exc import HookExecutionError
 from pre_commit.main import main as pre_commit
@@ -12,24 +14,19 @@ def test_pre_commit_hook_sync_nbstripout(
     tmp_repo,
     jupytext_repo_root,
     jupytext_repo_rev,
+    jupytext_pre_commit_config,
     notebook_with_outputs,
 ):
     """Here we sync the ipynb notebook with a Markdown file and also apply nbstripout."""
-    pre_commit_config_yaml = f"""
-repos:
-- repo: {jupytext_repo_root}
-  rev: {jupytext_repo_rev}
-  hooks:
-  - id: jupytext
-    args: [--sync]
-
-- repo: https://github.com/kynan/nbstripout
-  rev: 0.5.0
-  hooks:
-  - id: nbstripout
-"""
-
-    tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = ["--sync"]
+    jupytext_pre_commit_config["repos"].append({
+        "repo": "https://github.com/kynan/nbstripout",
+        "rev": "0.5.0",
+        "hooks": [{"id": "nbstripout"}]
+    })
+
+    with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), 'w') as file:
+        yaml.dump(jupytext_pre_commit_config, file)
 
     tmp_repo.git.add(".pre-commit-config.yaml")
     pre_commit(["install", "--install-hooks", "-f"])

tests/external/pre_commit/test_pre_commit_3_sync_black_nbstripout.py

@@ -1,3 +1,5 @@
+import os
+import yaml
 import pytest
 from git.exc import HookExecutionError
 from pre_commit.main import main as pre_commit
@@ -11,30 +13,31 @@ def test_pre_commit_hook_sync_black_nbstripout(
     tmp_repo,
     jupytext_repo_root,
     jupytext_repo_rev,
+    jupytext_pre_commit_config,
     notebook_with_outputs,
 ):
     """Here we sync the ipynb notebook with a py:percent file and also apply black and nbstripout."""
-    pre_commit_config_yaml = f"""
-repos:
-- repo: {jupytext_repo_root}
-  rev: {jupytext_repo_rev}
-  hooks:
-  - id: jupytext
-    args: [--sync, --pipe, black]
-    additional_dependencies:
-    - black==22.3.0  # Matches hook
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = [
+      "--sync", "--pipe", "black",
+    ]
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["additional_dependencies"] = [
+      "black==22.3.0",
+    ]
+    # Use python as language as we will need to install additional dependencies
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["language"] = "python"
+    jupytext_pre_commit_config["repos"].append({
+        "repo": "https://github.com/kynan/nbstripout",
+        "rev": "0.5.0",
+        "hooks": [{"id": "nbstripout"}]
+    })
+    jupytext_pre_commit_config["repos"].append({
+        "repo": "https://github.com/psf/black",
+        "rev": "22.3.0",
+        "hooks": [{"id": "black"}]
+    })
 
-- repo: https://github.com/psf/black
-  rev: 22.3.0
-  hooks:
-  - id: black
-
-- repo: https://github.com/kynan/nbstripout
-  rev: 0.5.0
-  hooks:
-  - id: nbstripout
-"""
-    tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
+    with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), 'w') as file:
+        yaml.dump(jupytext_pre_commit_config, file)
 
     tmp_repo.git.add(".pre-commit-config.yaml")
     pre_commit(["install", "--install-hooks", "-f"])

tests/external/pre_commit/test_pre_commit_4_sync_execute.py

@@ -1,3 +1,5 @@
+import os
+import yaml
 import pytest
 from git.exc import HookExecutionError
 from nbformat.v4.nbbase import new_code_cell
@@ -13,21 +15,22 @@ def test_pre_commit_hook_sync_execute(
     tmp_repo,
     jupytext_repo_root,
     jupytext_repo_rev,
+    jupytext_pre_commit_config,
     notebook_with_outputs,
 ):
     """Here we sync the ipynb notebook with a py:percent file and execute it (this is probably not a very
     recommendable hook!)"""
-    pre_commit_config_yaml = f"""
-repos:
-- repo: {jupytext_repo_root}
-  rev: {jupytext_repo_rev}
-  hooks:
-  - id: jupytext
-    args: [--sync, --execute, --show-changes]
-    additional_dependencies:
-    - nbconvert
-"""
-    tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = [
+        "--sync", "--execute", "--show-changes",
+    ]
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["additional_dependencies"] = [
+        "nbconvert"
+    ]
+    # Use python as language as we will need to install additional dependencies
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["language"] = "python"
+
+    with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), 'w') as file:
+        yaml.dump(jupytext_pre_commit_config, file)
 
     tmp_repo.git.add(".pre-commit-config.yaml")
     pre_commit(["install", "--install-hooks", "-f"])

tests/external/pre_commit/test_pre_commit_5_reformat_markdown.py

@@ -1,3 +1,5 @@
+import os
+import yaml
 import pytest
 from git.exc import HookExecutionError
 from nbformat.v4.nbbase import new_code_cell, new_markdown_cell, new_notebook
@@ -13,6 +15,7 @@ def test_pre_commit_hook_sync_reformat_code_and_markdown(
     tmp_repo,
     jupytext_repo_root,
     jupytext_repo_rev,
+    jupytext_pre_commit_config,
     notebook_with_outputs,
 ):
     """Here we sync the ipynb notebook with a py:percent file and also apply black and pandoc to reformat both
@@ -22,27 +25,32 @@ def test_pre_commit_hook_sync_reformat_code_and_markdown(
     ipynb files. Consequently we pin the version of nbformat to 5.0.8 in all the environments below (and you
     will have to do the same on the environment in which you edit the notebooks).
     """
-    pre_commit_config_yaml = f"""
-repos:
-- repo: {jupytext_repo_root}
-  rev: {jupytext_repo_rev}
-  hooks:
-  - id: jupytext
-    args: [--sync, --pipe-fmt, ipynb, --pipe, 'pandoc --from ipynb --to ipynb --markdown-headings=atx', --show-changes]
-    additional_dependencies:
-    - nbformat==5.0.8  # because pandoc 2.11.4 does not preserve yet the new cell ids
-  - id: jupytext
-    args: [--sync, --pipe, black, --show-changes]
-    additional_dependencies:
-    - black==22.3.0  # Matches black hook below
-    - nbformat==5.0.8  # for compatibility with the pandoc hook above
-
-- repo: https://github.com/psf/black
-  rev: 22.3.0
-  hooks:
-  - id: black
-"""
-    tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["args"] = [
+        "--sync", "--pipe-fmt", "ipynb", "--pipe", "pandoc", "--from", 
+        "ipynb", "--to", "ipynb", "--markdown-headings=atx", "--show-changes",
+    ]
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["additional_dependencies"] = [
+        "nbconvert==5.0.8"
+    ]
+    jupytext_pre_commit_config["repos"][0]["hooks"][1] = jupytext_pre_commit_config["repos"][0]["hooks"][0]
+    jupytext_pre_commit_config["repos"][0]["hooks"][1]["args"] = [
+        "--sync", "--pipe", "black", "--show-changes",
+    ]
+    jupytext_pre_commit_config["repos"][0]["hooks"][1]["additional_dependencies"] = [
+        "black==22.3.0",  # Matches black hook below
+        "nbformat==5.0.8",  # for compatibility with the pandoc hook above
+    ]
+    # Use python as language as we will need to install additional dependencies
+    jupytext_pre_commit_config["repos"][0]["hooks"][0]["language"] = "python"
+    jupytext_pre_commit_config["repos"][0]["hooks"][1]["language"] = "python"
+    jupytext_pre_commit_config["repos"][1] = {
+        "repo": "https://github.com/psf/black",
+        "rev": "22.3.0",
+        "hooks": [{"id": "black"}]
+    }
+
+    with open(os.path.join(tmpdir, ".pre-commit-config.yaml"), 'w') as file:
+        yaml.dump(jupytext_pre_commit_config, file)
 
     tmp_repo.git.add(".pre-commit-config.yaml")
     pre_commit(["install", "--install-hooks", "-f"])