Merge pull request #187 from andreytaboola/master

Add support of keeping the local context of the python scripts and notebooks by preserving working directory

Author: jonbannister

.circleci/config.yml

@@ -220,7 +220,7 @@ jobs:
       PYTHON_VERSION: "3_6"
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_6
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_6
-      VERSION: 0.6.2
+      VERSION: 0.6.3
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
       YARN_STATIC_DIR: notebooker/web/static/
       IMAGE_NAME: mangroup/notebooker
@@ -236,7 +236,7 @@ jobs:
     environment:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_7
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_7
-      VERSION: 0.6.2
+      VERSION: 0.6.3
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
       YARN_STATIC_DIR: notebooker/web/static/
       IMAGE_NAME: mangroup/notebooker
@@ -250,7 +250,7 @@ jobs:
     environment:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_8
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_8
-      VERSION: 0.6.2
+      VERSION: 0.6.3
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
       YARN_STATIC_DIR: notebooker/web/static/
       IMAGE_NAME: mangroup/notebooker
@@ -264,7 +264,7 @@ jobs:
     environment:
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_11
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_11
-      VERSION: 0.6.2
+      VERSION: 0.6.3
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
       YARN_STATIC_DIR: notebooker/web/static/
       IMAGE_NAME: mangroup/notebooker

CHANGELOG.md

@@ -1,3 +1,7 @@
+0.6.3 (2024-07-11)
+------------------
+* Feature: Flag to preserve original working directory when running notebooks to make local imports and relative paths work.
+
 0.6.2 (2024-05-02)
 ------------------
 * Bugfix: Folders with spaces in their names are now correctly handled in the webapp.

docs/conf.py

@@ -23,7 +23,7 @@
 author = "Man Group Quant Tech"
 
 # The full version, including alpha/beta/rc tags
-release = "0.6.2"
+release = "0.6.3"
 
 
 # -- General configuration ---------------------------------------------------

notebooker/_entrypoints.py

@@ -64,6 +64,12 @@ def filesystem_default_value(dirname):
     is_flag=True,
     help="If selected, notebooker will not try to pull the latest version of python templates from git.",
 )
+@click.option(
+    "--execute-at-origin",
+    default=False,
+    is_flag=True,
+    help="If selected, notebooker set current working directory to absolute path of the notebook to keep it local context available",
+)
 @click.option(
     "--default-mailfrom", default=DEFAULT_MAILFROM_ADDRESS, help="Set a new value for the default mailfrom setting."
 )
@@ -84,6 +90,7 @@ def base_notebooker(
     py_template_base_dir,
     py_template_subdir,
     notebooker_disable_git,
+    execute_at_origin,
     default_mailfrom,
     running_timeout,
     serializer_cls,
@@ -98,6 +105,7 @@ def base_notebooker(
         PY_TEMPLATE_BASE_DIR=py_template_base_dir,
         PY_TEMPLATE_SUBDIR=py_template_subdir,
         NOTEBOOKER_DISABLE_GIT=notebooker_disable_git,
+        EXECUTE_AT_ORIGIN=execute_at_origin,
         DEFAULT_MAILFROM=default_mailfrom,
         RUNNING_TIMEOUT=running_timeout,
     )

notebooker/execute_notebook.py

@@ -48,6 +48,7 @@ def _run_checks(
     email_subject: Optional[str] = "",
     prepare_only: Optional[bool] = False,
     notebooker_disable_git: bool = False,
+    execute_at_origin: bool = False,
     py_template_base_dir: str = "",
     py_template_subdir: str = "",
     scheduler_job_id: Optional[str] = None,
@@ -82,6 +83,10 @@ def _run_checks(
         Comma-separated email addresses to send on error.
     prepare_only : `Optional[bool]`
         Internal usage. Whether we want to do everything apart from executing the notebook.
+    notebooker_disable_git : `bool`
+        Whether to disable git pulling of the notebook templates.
+    execute_at_origin : `bool`
+        Whether to execute the notebook at the original path of the template notebook.
     scheduler_job_id : `Optional[str]`
         If available, it will be part of the Error or Completed run report.
     mailfrom : `Optional[str]`
@@ -111,8 +116,18 @@ def _run_checks(
     ipynb_executed_path = os.path.join(output_dir, output_ipynb)
 
     logger.info("Executing notebook at {} using parameters {} --> {}".format(ipynb_raw_path, overrides, output_ipynb))
+    working_dir = None
+    if execute_at_origin:
+        working_dir = os.path.dirname(os.path.join(py_template_dir, template_name))
+        logger.info("Setting working directory for execution {}".format(working_dir))
+
     pm.execute_notebook(
-        ipynb_raw_path, ipynb_executed_path, parameters=overrides, log_output=True, prepare_only=prepare_only
+        ipynb_raw_path,
+        ipynb_executed_path,
+        parameters=overrides,
+        log_output=True,
+        prepare_only=prepare_only,
+        cwd=working_dir,
     )
     with open(ipynb_executed_path, "r") as f:
         raw_executed_ipynb = f.read()
@@ -163,6 +178,7 @@ def run_report(
     hide_code=False,
     prepare_only=False,
     notebooker_disable_git=False,
+    execute_at_origin=False,
     py_template_base_dir="",
     py_template_subdir="",
     scheduler_job_id=None,
@@ -200,6 +216,7 @@ def run_report(
             hide_code=hide_code,
             prepare_only=prepare_only,
             notebooker_disable_git=notebooker_disable_git,
+            execute_at_origin=execute_at_origin,
             py_template_base_dir=py_template_base_dir,
             py_template_subdir=py_template_subdir,
             scheduler_job_id=scheduler_job_id,
@@ -353,6 +370,7 @@ def execute_notebook_entrypoint(
     output_dir, template_dir, _ = initialise_base_dirs(output_dir=config.OUTPUT_DIR, template_dir=config.TEMPLATE_DIR)
     all_overrides = _get_overrides(overrides_as_json, iterate_override_values_of)
     notebooker_disable_git = config.NOTEBOOKER_DISABLE_GIT
+    execute_at_origin = config.EXECUTE_AT_ORIGIN
     py_template_base_dir = config.PY_TEMPLATE_BASE_DIR
     py_template_subdir = config.PY_TEMPLATE_SUBDIR
 
@@ -376,6 +394,7 @@ def execute_notebook_entrypoint(
     logger.info("prepare_notebook_only = %s", prepare_notebook_only)
     logger.info("scheduler job id = %s", scheduler_job_id)
     logger.info("notebooker_disable_git = %s", notebooker_disable_git)
+    logger.info("execute_at_origin = %s", execute_at_origin)
     logger.info("py_template_base_dir = %s", py_template_base_dir)
     logger.info("py_template_subdir = %s", py_template_subdir)
     logger.info("serializer_cls = %s", config.SERIALIZER_CLS)
@@ -402,6 +421,7 @@ def execute_notebook_entrypoint(
             hide_code=hide_code,
             prepare_only=prepare_notebook_only,
             notebooker_disable_git=notebooker_disable_git,
+            execute_at_origin=execute_at_origin,
             py_template_base_dir=py_template_base_dir,
             py_template_subdir=py_template_subdir,
             scheduler_job_id=scheduler_job_id,
@@ -532,6 +552,7 @@ def run_report_in_subprocess(
             base_config.DEFAULT_MAILFROM,
         ]
         + (["--notebooker-disable-git"] if base_config.NOTEBOOKER_DISABLE_GIT else [])
+        + (["--execute-at-origin"] if base_config.EXECUTE_AT_ORIGIN else [])
         + ["--serializer-cls", result_serializer.__class__.__name__]
         + result_serializer.serializer_args_to_cmdline_args()
         + [

notebooker/settings.py

@@ -25,6 +25,9 @@ class BaseConfig:
     # or list the available templates.
     NOTEBOOKER_DISABLE_GIT: bool = False
 
+    # A boolean flag to dictate whether we should execute the notebook at the origin or not.
+    EXECUTE_AT_ORIGIN: bool = False
+
     # The serializer class we are using for storage, e.g. PyMongoResultSerializer
     SERIALIZER_CLS: DEFAULT_SERIALIZER = None
     # The dictionary of parameters which are used to initialize the serializer class above

notebooker/version.py

@@ -1 +1 @@
-__version__ = "0.6.2"
+__version__ = "0.6.3"

notebooker/web/static/package.json

@@ -1,6 +1,6 @@
 {
   "name": "notebooker",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "Notebooker - Turn notebooks into reports",
   "dependencies": {
     "bootstrap-table": "1.20.2",

tests/regression/local_context/__init__.py

@@ -0,0 +1,2 @@
+Hello, World!
+This is a sample text file.