Give wptrunner a distinct exit code for CRITICAL errors

Previously, when running tests without --stability, wptrunner would
exit with 1 if there were unexpected results or if there was a
critical log message. With --stability, wptrunner would exit with
either 1 or 2 for stability failures, and 1 if the stability check
succeeded but there were critical log messages. In either case, wpt
run would then simplify this to always exiting with 0 or 1.

We now reserve exit codes above 64 for wptrunner's own usage, and exit
with 64 if there is a critical log message. This now takes priority
over the original exit code.

We also now allow wpt run to exit with the exit code that wptrunner
returns.

Author: gsnedders

tools/wpt/run.py

@@ -1010,7 +1010,7 @@ def run(venv, **kwargs):
     setup_cls, wptrunner_kwargs = setup_wptrunner(venv, **kwargs)
 
     try:
-        rv = run_single(venv, **wptrunner_kwargs) > 0
+        rv = run_single(venv, **wptrunner_kwargs)
     finally:
         setup_cls.teardown()
 

tools/wptrunner/wptrunner/wptrunner.py

@@ -568,14 +568,14 @@ def check_stability(**kwargs):
                                      **kwargs)
 
 
-def start(**kwargs):
+def start(**kwargs: Any) -> int:
     assert logger is not None
 
     logged_critical = wptlogging.LoggedAboveLevelHandler("CRITICAL")
     handler = handlers.LogLevelFilter(logged_critical, "CRITICAL")
     logger.add_handler(handler)
 
-    rv = False
+    rv = 0
     try:
         if kwargs["list_test_groups"]:
             list_test_groups(**kwargs)
@@ -586,12 +586,19 @@ def start(**kwargs):
         elif kwargs["list_tests_json"]:
             list_tests_json(**kwargs)
         elif kwargs["verify"] or kwargs["stability"]:
-            rv = check_stability(**kwargs) or logged_critical.has_log
+            rv = check_stability(**kwargs)
         else:
-            rv = not run_tests(**kwargs)[0] or logged_critical.has_log
+            rv = not run_tests(**kwargs)[0]
     finally:
         logger.shutdown()
         logger.remove_handler(handler)
+
+    # Reserve everything above 64 for our global usage.
+    assert 0 <= rv < 64, "Exit codes above 64 are reserved"
+    if logged_critical.has_log:
+        print("Did log critical")
+        rv = 64
+
     return rv