Merge pull request #85 from man-group/fix-37-master

Fix for py37 build.

Author: jonbannister

.circleci/config.yml

@@ -81,7 +81,7 @@ defaults: &defaults
          virtualenv ci
          . ci/bin/activate
          pip install six lxml flake8 tox pytest black .[test]
-         python setup.py develop
+         pip install --editable .
     # Save dependency cache
     - save_cache:
         key: v1-dep-{{ .Branch }}-{{ epoch }}

CONTRIBUTING.md

@@ -1,7 +1,7 @@
 # Dev environment setup
 
 Dev environment setup is largely the same as setup in the tutorial, but instead of pip installing the version
-in pypi, you can set up using `python setup.py develop`.
+in pypi, you can set up using `pip install --editable .`.
 
 
 # Contributing

notebooker/constants.py

@@ -148,7 +148,7 @@ class NotebookResultComplete(NotebookResultBase):
     raw_html = attr.ib(default="")
     email_html = attr.ib(default="")
     update_time = attr.ib(default=datetime.datetime.now())
-    pdf = attr.ib(default="")
+    pdf = attr.ib(default=b"")
     report_title = attr.ib(default="")
     overrides = attr.ib(default=attr.Factory(dict))
     mailto = attr.ib(default="")

notebooker/execute_notebook.py

@@ -110,7 +110,7 @@ def _run_checks(
 
     logger.info("Saving output notebook as HTML from {}".format(ipynb_executed_path))
     html, resources = ipython_to_html(ipynb_executed_path, job_id)
-    email_html, resources = ipython_to_html(ipynb_executed_path, job_id, hide_code=hide_code)
+    email_html, _ = ipython_to_html(ipynb_executed_path, job_id, hide_code=hide_code)
     pdf = ipython_to_pdf(raw_executed_ipynb, report_title, hide_code=hide_code) if generate_pdf_output else ""
 
     notebook_result = NotebookResultComplete(

notebooker/serialization/mongo.py

@@ -199,16 +199,26 @@ def _convert_result(
         if cls is None:
             return None
 
+        def ignore_missing_files(f):
+            def _ignore_missing_files(path, *args, **kwargs):
+                try:
+                    return f(path, *args, **kwargs)
+                except NoFile:
+                    logger.error("Could not find file %s in %s", path, self.result_data_store)
+                    return ""
+            return _ignore_missing_files
+
+        @ignore_missing_files
         def read_file(path, is_json=False):
+            r = self.result_data_store.get_last_version(path).read()
             try:
-                r = self.result_data_store.get_last_version(path).read()
-                try:
-                    return "" if not r else json.loads(r) if is_json else r.decode("utf8")
-                except UnicodeDecodeError:
-                    return r
-            except NoFile:
-                logger.error("Could not find file %s in %s", path, self.result_data_store)
-                return ""
+                return "" if not r else json.loads(r) if is_json else r.decode("utf8")
+            except UnicodeDecodeError:
+                return r
+
+        @ignore_missing_files
+        def read_bytes_file(path):
+            return self.result_data_store.get_last_version(path).read()
 
         if not load_payload:
             result.pop("stdout", None)
@@ -219,7 +229,7 @@ def read_file(path, is_json=False):
                 result["raw_html_resources"]["outputs"] = outputs
                 if result.get("generate_pdf_output"):
                     pdf_filename = _pdf_filename(result["job_id"])
-                    result["pdf"] = read_file(pdf_filename)
+                    result["pdf"] = read_bytes_file(pdf_filename)
                 if not result.get("raw_ipynb_json"):
                     result["raw_ipynb_json"] = read_file(_raw_json_filename(result["job_id"]), is_json=True)
                 if not result.get("raw_html"):

notebooker/utils/conversion.py

@@ -33,6 +33,7 @@ def ipython_to_html(ipynb_path: str, job_id: str, hide_code: bool = False) -> (n
         nb = nbformat.reads(nb_file.read(), as_version=nbformat.v4.nbformat)
     resources_dir = get_resources_dir(job_id)
     html, resources = html_exporter_with_figs.from_notebook_node(nb, resources={"output_files_dir": resources_dir})
+    resources = {k: v for (k, v) in resources.items() if not callable(v)}
     return html, resources
 
 

setup.py

@@ -51,28 +51,29 @@ def get_long_description():
     zip_safe=False,
     include_package_data=True,
     install_requires=[
+        "apscheduler",
+        "babel",
+        "cachelib",
+        "click>7.1.0",
+        "dataclasses",
+        "flask",
         "gevent",
+        "gitpython",
+        "inflection",
+        "ipykernel",
         "ipython",
-        "pandas",
+        "ipython_genutils",
+        "jupytext>=1.2.0",
         "matplotlib",
-        "pymongo",
-        "papermill",
-        "dataclasses",
-        "nbconvert<6.0.0",  # Pin this because new template locations do not seem to work on OSX
+        "nbconvert",
         "nbformat",
-        "jupytext>=1.2.0",
-        "ipykernel",
-        "stashy",
-        "click>7.1.0",
+        "pandas",
+        "papermill",
+        "pymongo",
         "python-dateutil",
-        "apscheduler",
-        "flask",
         "requests",
         "retrying",
-        "gitpython",
-        "cachelib",
-        "inflection",
-        "babel",
+        "stashy",
     ],
     extras_require={
         "prometheus": ["prometheus_client"],

tests/conftest.py

@@ -1,4 +1,9 @@
+import os
+
 import pytest
+from pytest_server_fixtures import CONFIG
+from pytest_server_fixtures.mongo import MongoTestServer
+from pytest_fixture_config import yield_requires_config
 
 from notebooker.constants import (
     DEFAULT_DATABASE_NAME,
@@ -11,6 +16,43 @@
 from notebooker.web.app import create_app, setup_app
 
 
+class MongoTestServerWithPath(MongoTestServer):
+    @property
+    def env(self):
+        return {"PATH": os.environ["PATH"]}
+
+
+def _mongo_server():
+    """This does the actual work - there are several versions of this used
+    with different scopes.
+    """
+    test_server = MongoTestServerWithPath()
+    try:
+        test_server.start()
+        yield test_server
+    finally:
+        test_server.teardown()
+
+
+@pytest.yield_fixture(scope="function")
+@yield_requires_config(CONFIG, ["mongo_bin"])
+def mongo_server():
+    """Function-scoped MongoDB server started in a local thread.
+    This also provides a temp workspace.
+    We tear down, and cleanup mongos at the end of the test.
+
+    For completeness, we tidy up any outstanding mongo temp directories
+    at the start and end of each test session
+
+    Attributes
+    ----------
+    api (`pymongo.MongoClient`)  : PyMongo Client API connected to this server
+    .. also inherits all attributes from the `workspace` fixture
+    """
+    for server in _mongo_server():
+        yield server
+
+
 @pytest.fixture
 def bson_library(mongo_server, test_db_name, test_lib_name):
     return mongo_server.api[test_db_name][test_lib_name]
@@ -98,7 +140,7 @@ def flask_app(webapp_config):
 
 
 @pytest.fixture
-def setup_and_cleanup_notebooker_filesystem(webapp_config):
+def setup_and_cleanup_notebooker_filesystem(webapp_config, setup_workspace):
     try:
         initialise_base_dirs(webapp_config=webapp_config)
         yield

tests/integration/conftest.py

@@ -117,6 +117,7 @@
 }
 """
 
+
 @pytest.fixture
 def setup_workspace(workspace):
     (workspace.workspace + "/templates").mkdir()
@@ -130,4 +131,4 @@ def setup_workspace(workspace):
     ipynb_report_to_run.write_lines(DUMMY_REPORT_IPYNB.split("\n"))
 
     report_to_run_failing = workspace.workspace + "/templates/fake/report_failing.py"
-    report_to_run_failing.write_lines(DUMMY_FAILING_REPORT.split("\n"))
+    report_to_run_failing.write_lines(DUMMY_FAILING_REPORT.split("\n"))

tests/integration/test_e2e.py

@@ -32,10 +32,7 @@ def _check_report_output(job_id, serialiser, **kwargs):
 
 @pytest.mark.parametrize(
     "report_name",
-    [
-        "fake/py_report",
-        "fake/ipynb_report"
-    ],
+    ["fake/py_report", "fake/ipynb_report"],
 )
 @freezegun.freeze_time(datetime.datetime(2018, 1, 12))
 def test_run_report(bson_library, flask_app, setup_and_cleanup_notebooker_filesystem, setup_workspace, report_name):
@@ -66,6 +63,7 @@ def test_run_report(bson_library, flask_app, setup_and_cleanup_notebooker_filesy
         assert job_id == serialiser.get_latest_successful_job_id_for_name_and_params(report_name, overrides)
         assert job_id == serialiser.get_latest_successful_job_id_for_name_and_params(report_name, None)
 
+
 @freezegun.freeze_time(datetime.datetime(2018, 1, 12))
 def test_run_failing_report(bson_library, flask_app, setup_and_cleanup_notebooker_filesystem, setup_workspace):
     with flask_app.app_context():
@@ -91,13 +89,12 @@ def test_run_failing_report(bson_library, flask_app, setup_and_cleanup_notebooke
 
 @pytest.mark.parametrize(
     "report_name",
-    [
-        "fake/py_report",
-        "fake/ipynb_report"
-    ],
+    ["fake/py_report", "fake/ipynb_report"],
 )
 @freezegun.freeze_time(datetime.datetime(2018, 1, 12))
-def test_run_report_and_rerun(bson_library, flask_app, setup_and_cleanup_notebooker_filesystem, setup_workspace, report_name):
+def test_run_report_and_rerun(
+    bson_library, flask_app, setup_and_cleanup_notebooker_filesystem, setup_workspace, report_name
+):
     with flask_app.app_context():
         serialiser = get_serializer()
         overrides = {"n_points": 5}
@@ -140,13 +137,12 @@ def test_run_report_and_rerun(bson_library, flask_app, setup_and_cleanup_noteboo
 
 @pytest.mark.parametrize(
     "report_name",
-    [
-        "fake/py_report",
-        "fake/ipynb_report"
-    ],
+    ["fake/py_report", "fake/ipynb_report"],
 )
 @freezegun.freeze_time(datetime.datetime(2018, 1, 12))
-def test_run_report_hide_code(bson_library, flask_app, setup_and_cleanup_notebooker_filesystem, setup_workspace, report_name):
+def test_run_report_hide_code(
+    bson_library, flask_app, setup_and_cleanup_notebooker_filesystem, setup_workspace, report_name
+):
     with flask_app.app_context():
         serialiser = get_serializer()
         overrides = {"n_points": 5}