Increase test coverage for :fail-fast?

This will help reliably coming up with a fix for #75.

Coverage is extended by:

* providing counterexamples for each test case
* also exercising exception handling, whether it happens in the deftest body, or in fixtures.

Author: vemv

eftest/test/eftest/runner_test.clj

@@ -2,16 +2,42 @@
   (:require [clojure.test :refer :all]
             [eftest.runner :as sut]))
 
-(in-ns 'eftest.test-ns.single-failing-test)
+(def tests-completed
+  (atom []))
+
+;; Example test namespaces have a prefix such as ns-0 so that they can be
+;; deterministically sorted.
+
+(in-ns 'eftest.test-ns-0.throwing-test)
+(clojure.core/refer-clojure)
+(clojure.core/require 'clojure.test)
+(clojure.test/deftest throwing-test
+  (swap! eftest.runner-test/tests-completed conj :throwing-test)
+  (throw (ex-info "." {})))
+
+(in-ns 'eftest.test-ns-1.throwing-test-in-fixtures)
+(clojure.core/refer-clojure)
+(clojure.core/require 'clojure.test)
+(clojure.test/use-fixtures :once
+  (fn [t]
+    (swap! eftest.runner-test/tests-completed conj :throwing-test-in-fixtures)
+    (throw (ex-info "." {}))
+    (t)))
+(clojure.test/deftest example-test
+  (clojure.test/is (= 1 1)))
+
+(in-ns 'eftest.test-ns-2.single-failing-test)
 (clojure.core/refer-clojure)
 (clojure.core/require 'clojure.test)
 (clojure.test/deftest single-failing-test
+  (swap! eftest.runner-test/tests-completed conj :single-failing-test)
   (clojure.test/is (= 1 2)))
 
-(in-ns 'eftest.test-ns.another-failing-test)
+(in-ns 'eftest.test-ns-3.another-failing-test)
 (clojure.core/refer-clojure)
 (clojure.core/require 'clojure.test)
 (clojure.test/deftest another-failing-test
+  (swap! eftest.runner-test/tests-completed conj :another-failing-test)
   (clojure.test/is (= 3 4)))
 
 (in-ns 'eftest.test-ns.slow-test)
@@ -44,23 +70,70 @@
       :return @ret})))
 
 (deftest test-reporting
-  (let [out (:output (test-run-tests 'eftest.test-ns.single-failing-test))]
-    (is (re-find #"FAIL in eftest.test-ns.single-failing-test/single-failing-test" out))
+  (let [out (:output (test-run-tests 'eftest.test-ns-2.single-failing-test))]
+    (is (re-find #"FAIL in eftest.test-ns-2.single-failing-test/single-failing-test" out))
     (is (not (re-find #"IllegalArgumentException" out)))))
 
 (deftest test-fail-fast
-  (let [result (:return
-                (test-run-tests
-                 '[eftest.test-ns.single-failing-test
-                   eftest.test-ns.another-failing-test]
-                 {:fail-fast? true, :multithread? false}))]
-    (is (= {:test 1 :fail 1} (select-keys result [:test :fail])))))
+  (let [run! (fn [nses opts]
+               (reset! tests-completed [])
+               (-> nses
+                   (test-run-tests (merge {:multithread? false} opts))
+                   :return
+                   (select-keys [:test :fail :error])))]
+
+    (testing "Failing tests"
+      (let [result (run! '[eftest.test-ns-2.single-failing-test
+                           eftest.test-ns-3.another-failing-test]
+                         {:fail-fast? true})]
+        (is (= {:test 1 :fail 1 :error 0} result))
+        (is (= [:single-failing-test]
+               @tests-completed)))
+
+      (let [result (run! '[eftest.test-ns-2.single-failing-test
+                           eftest.test-ns-3.another-failing-test]
+                         {:fail-fast? false})]
+        (is (= {:test 2 :fail 2 :error 0} result))
+        (is (= [:single-failing-test :another-failing-test]
+               @tests-completed))))
+
+    (testing "Exceptions in tests"
+      (let [result (run! '[eftest.test-ns-0.throwing-test
+                           eftest.test-ns-2.single-failing-test]
+                         {:fail-fast? true})]
+        (is (= {:test 1 :fail 0 :error 1} result))
+        (is (= [:throwing-test]
+               @tests-completed)))
+
+      (let [result (run! '[eftest.test-ns-0.throwing-test
+                           eftest.test-ns-2.single-failing-test]
+                         {:fail-fast? false})]
+        (is (= {:test 2 :fail 1 :error 1} result))
+        (is (= [:throwing-test
+                :single-failing-test]
+               @tests-completed))))
+
+    (testing "Exceptions in fixtures"
+      (let [result (run! '[eftest.test-ns-1.throwing-test-in-fixtures
+                           eftest.test-ns-2.single-failing-test]
+                         {:fail-fast? true})]
+        (is (= {:test 0 :fail 0 :error 1} result))
+        (is (= [:throwing-test-in-fixtures]
+               @tests-completed)))
+
+      (let [result (run! '[eftest.test-ns-1.throwing-test-in-fixtures
+                           eftest.test-ns-2.single-failing-test]
+                         {:fail-fast? false})]
+        (is (= {:test 1 :fail 1 :error 1} result))
+        (is (= [:throwing-test-in-fixtures
+                :single-failing-test]
+               @tests-completed))))))
 
 (deftest test-fail-multi
   (let [out (:output
              (test-run-tests
-              '[eftest.test-ns.single-failing-test
-                eftest.test-ns.another-failing-test]))]
+              '[eftest.test-ns-2.single-failing-test
+                eftest.test-ns-3.another-failing-test]))]
     (println out)
     (is (re-find #"(?m)expected: 1\n  actual: 2" out))
     (is (re-find #"(?m)expected: 3\n  actual: 4" out))))