feat: Record & upload test failures in CI.

BenHenning · BenHenning · commit 27b375fca063 · 2025-11-21T03:18:15.000Z
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -69,11 +69,18 @@ jobs:
       - name: Run tests (up to 50 times)
         run: |
           cd main/test/webdriverio/test
-          for i in $(seq 1 50);
-          do
+          #for i in $(seq 1 50);
+          #do
             echo "Attempt $i"
             npx mocha --timeout 30000 dist
-          done
+          #done
+
+      - name: Upload test failure screenshots
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: test-failure-screenshots
+          path: test/webdriverio/test/failures
 
   webdriverio_tests:
     name: WebdriverIO tests (against pinned v12)
diff --git a/package.json b/package.json
@@ -22,8 +22,8 @@
     "wdio:build": "npm run wdio:build:app && npm run wdio:build:tests",
     "wdio:build:app": "cd test/webdriverio && webpack",
     "wdio:build:tests": "tsc -p ./test/webdriverio/test/tsconfig.json",
-    "wdio:clean": "cd test/webdriverio/test && rm -rf dist",
-    "wdio:run": "npm run wdio:build && cd test/webdriverio/test && npx mocha dist",
+    "wdio:clean": "cd test/webdriverio/test && rm -rf dist && rm -rf failures",
+    "wdio:run": "npm run wdio:build && cd test/webdriverio/test && mkdir failures && npx mocha dist",
     "wdio:run:ci": "npm run wdio:build && cd test/webdriverio/test && npx mocha --timeout 30000 dist"
   },
   "main": "./dist/index.js",
diff --git a/test/webdriverio/test/actions_test.ts b/test/webdriverio/test/actions_test.ts
@@ -20,6 +20,7 @@ import {
   sendKeyAndWait,
   keyRight,
   contextMenuItems,
+  checkForFailures,
 } from './test_setup.js';
 
 const isDarwin = process.platform === 'darwin';
@@ -99,6 +100,10 @@ suite('Menus test', function () {
     await this.browser.pause(PAUSE_TIME);
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Menu action via keyboard on block opens menu', async function () {
     // Navigate to draw_circle_1.
     await focusOnBlock(this.browser, 'draw_circle_1');
diff --git a/test/webdriverio/test/basic_test.ts b/test/webdriverio/test/basic_test.ts
@@ -22,6 +22,7 @@ import {
   keyRight,
   keyUp,
   keyDown,
+  checkForFailures,
 } from './test_setup.js';
 import {Key} from 'webdriverio';
 
@@ -37,6 +38,10 @@ suite('Keyboard navigation on Blocks', function () {
     );
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Default workspace', async function () {
     const blockCount = await this.browser.execute(() => {
       return Blockly.getMainWorkspace().getAllBlocks(false).length;
@@ -235,6 +240,10 @@ suite('Keyboard navigation on Fields', function () {
     );
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Up from first field selects block', async function () {
     await focusOnBlockField(this.browser, 'p5_canvas_1', 'WIDTH');
     await this.browser.pause(PAUSE_TIME);
diff --git a/test/webdriverio/test/block_comment_test.ts b/test/webdriverio/test/block_comment_test.ts
@@ -14,6 +14,7 @@ import {
   testFileLocations,
   keyRight,
   PAUSE_TIME,
+  checkForFailures,
 } from './test_setup.js';
 import {Key} from 'webdriverio';
 
@@ -34,6 +35,10 @@ suite('Block comment navigation', function () {
     });
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Activating a block comment icon focuses the comment', async function () {
     await focusOnBlock(this.browser, 'p5_canvas_1');
     await keyRight(this.browser);
diff --git a/test/webdriverio/test/clipboard_test.ts b/test/webdriverio/test/clipboard_test.ts
@@ -18,6 +18,7 @@ import {
   blockIsPresent,
   getFocusedBlockType,
   sendKeyAndWait,
+  checkForFailures,
 } from './test_setup.js';
 import {Key, KeyAction, PointerAction, WheelAction} from 'webdriverio';
 
@@ -31,6 +32,10 @@ suite('Clipboard test', function () {
     await this.browser.pause(PAUSE_TIME);
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Copy and paste while block selected', async function () {
     // Navigate to draw_circle_1.
     await focusOnBlock(this.browser, 'draw_circle_1');
diff --git a/test/webdriverio/test/delete_test.ts b/test/webdriverio/test/delete_test.ts
@@ -18,6 +18,7 @@ import {
   sendKeyAndWait,
   keyRight,
   focusOnBlockField,
+  checkForFailures,
 } from './test_setup.js';
 import {Key} from 'webdriverio';
 
@@ -34,6 +35,10 @@ suite('Deleting Blocks', function () {
     await this.browser.pause(PAUSE_TIME);
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Deleting block selects parent block', async function () {
     await focusOnBlock(this.browser, 'controls_if_2');
     await this.browser.pause(PAUSE_TIME);
diff --git a/test/webdriverio/test/duplicate_test.ts b/test/webdriverio/test/duplicate_test.ts
@@ -15,6 +15,7 @@ import {
   testFileLocations,
   testSetup,
   sendKeyAndWait,
+  checkForFailures,
 } from './test_setup.js';
 
 suite('Duplicate test', function () {
@@ -27,6 +28,10 @@ suite('Duplicate test', function () {
     await this.browser.pause(PAUSE_TIME);
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Duplicate block', async function () {
     // Navigate to draw_circle_1.
     await focusOnBlock(this.browser, 'draw_circle_1');
diff --git a/test/webdriverio/test/flyout_test.ts b/test/webdriverio/test/flyout_test.ts
@@ -20,6 +20,7 @@ import {
   getCurrentFocusNodeId,
   getCurrentFocusedBlockId,
   tabNavigateToToolbox,
+  checkForFailures,
 } from './test_setup.js';
 
 suite('Toolbox and flyout test', function () {
@@ -32,6 +33,10 @@ suite('Toolbox and flyout test', function () {
     await this.browser.pause(PAUSE_TIME);
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Tab navigating to toolbox should open flyout', async function () {
     // Two tabs should navigate to the toolbox (initial element is skipped).
     await tabNavigateForward(this.browser);
diff --git a/test/webdriverio/test/insert_test.ts b/test/webdriverio/test/insert_test.ts
@@ -22,6 +22,7 @@ import {
   blockIsPresent,
   keyUp,
   tabNavigateToToolbox,
+  checkForFailures,
 } from './test_setup.js';
 
 suite('Insert test', function () {
@@ -34,6 +35,10 @@ suite('Insert test', function () {
     await this.browser.pause(PAUSE_TIME);
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Insert and cancel with block selection', async function () {
     // Navigate to draw_circle_1.
     await focusOnBlock(this.browser, 'draw_circle_1');
@@ -148,6 +153,10 @@ suite('Insert test with more blocks', function () {
     await this.browser.pause(PAUSE_TIME);
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Does not bump immovable input blocks on insert', async function () {
     // Focus the print block with a connected input block. Ordinarily, inserting
     // an input block would connect it to this block and bump its child, but
diff --git a/test/webdriverio/test/keyboard_mode_test.ts b/test/webdriverio/test/keyboard_mode_test.ts
@@ -15,6 +15,7 @@ import {
   tabNavigateToWorkspace,
   clickBlock,
   sendKeyAndWait,
+  checkForFailures,
 } from './test_setup.js';
 import {Key} from 'webdriverio';
 
@@ -48,6 +49,10 @@ suite(
       await tabNavigateToWorkspace(this.browser);
     });
 
+    teardown(async function() {
+      await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+    });
+
     test('T to open toolbox enables keyboard mode', async function () {
       await this.browser.pause(PAUSE_TIME);
       await sendKeyAndWait(this.browser, 't');
diff --git a/test/webdriverio/test/move_test.ts b/test/webdriverio/test/move_test.ts
@@ -16,6 +16,7 @@ import {
   sendKeyAndWait,
   keyDown,
   contextMenuItems,
+  checkForFailures,
 } from './test_setup.js';
 
 suite.skip('Move start tests', function () {
@@ -32,6 +33,10 @@ suite.skip('Move start tests', function () {
     await this.browser.pause(PAUSE_TIME);
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   // When a move of a statement block begins, it is expected that only
   // that block (and all blocks connected to its inputs) will be
   // moved, with subsequent statement blocks below it in the stack
@@ -194,6 +199,10 @@ suite('Statement move tests', function () {
     await this.browser.pause(PAUSE_TIME);
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   /** Serialized simple statement block with no statement inputs. */
   const STATEMENT_SIMPLE = {
     type: 'draw_emoji',
@@ -477,6 +486,10 @@ suite(`Value expression move tests`, function () {
         await this.browser.pause(PAUSE_TIME);
       });
 
+      teardown(async function() {
+        await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+      });
+
       suite('Constrained moves of a simple reporter block', function () {
         setup(async function () {
           await appendBlock(this.browser, VALUE_SIMPLE, 'join0', 'ADD0');
diff --git a/test/webdriverio/test/mutator_test.ts b/test/webdriverio/test/mutator_test.ts
@@ -17,6 +17,7 @@ import {
   sendKeyAndWait,
   keyRight,
   keyDown,
+  checkForFailures,
 } from './test_setup.js';
 import {Key} from 'webdriverio';
 
@@ -40,6 +41,10 @@ suite('Mutator navigation', function () {
     };
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Enter opens mutator', async function () {
     await this.openMutator();
 
diff --git a/test/webdriverio/test/scroll_test.ts b/test/webdriverio/test/scroll_test.ts
@@ -16,6 +16,7 @@ import {
   tabNavigateToWorkspace,
   testFileLocations,
   testSetup,
+  checkForFailures,
 } from './test_setup.js';
 
 suite('Scrolling into view', function () {
@@ -40,6 +41,10 @@ suite('Scrolling into view', function () {
     await testSetup(testFileLocations.BASE, this.timeout());
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Insert scrolls new block into view', async function () {
     // Increase timeout to 10s for this longer test.
     this.timeout(PAUSE_TIME ? 0 : 10000);
diff --git a/test/webdriverio/test/stack_navigation.ts b/test/webdriverio/test/stack_navigation.ts
@@ -13,6 +13,7 @@ import {
   testFileLocations,
   testSetup,
   sendKeyAndWait,
+  checkForFailures,
 } from './test_setup.js';
 
 suite('Stack navigation', function () {
@@ -22,6 +23,10 @@ suite('Stack navigation', function () {
     await this.browser.pause(PAUSE_TIME);
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Next', async function () {
     await tabNavigateToWorkspace(this.browser);
     chai.assert.equal(
diff --git a/test/webdriverio/test/styling_test.ts b/test/webdriverio/test/styling_test.ts
@@ -14,6 +14,7 @@ import {
   tabNavigateToWorkspace,
   testFileLocations,
   testSetup,
+  checkForFailures,
 } from './test_setup.js';
 import * as chai from 'chai';
 
@@ -27,6 +28,10 @@ suite('Styling test', function () {
     await this.browser.pause(PAUSE_TIME);
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   async function strokeColorEquals(
     browser: WebdriverIO.Browser,
     selector: string,
diff --git a/test/webdriverio/test/test_setup.ts b/test/webdriverio/test/test_setup.ts
@@ -128,6 +128,16 @@ export async function testSetup(
   return driver;
 }
 
+export async function checkForFailures(
+  browser: WebdriverIO.Browser,
+  testTitle: string,
+  testState: string | undefined
+) {
+  if (testState === 'failed') {
+    await browser.saveScreenshot(`failures/${testTitle}.png`);
+  }
+}
+
 /**
  * Replaces OS-specific path with POSIX style path.
  *
diff --git a/test/webdriverio/test/toast_test.ts b/test/webdriverio/test/toast_test.ts
@@ -6,7 +6,7 @@
 
 import * as chai from 'chai';
 import * as Blockly from 'blockly/core';
-import {PAUSE_TIME, testFileLocations, testSetup} from './test_setup.js';
+import {PAUSE_TIME, testFileLocations, testSetup, checkForFailures} from './test_setup.js';
 
 suite('HTML toasts', function () {
   // Disable timeouts when non-zero PAUSE_TIME is used to watch tests run.
@@ -18,6 +18,10 @@ suite('HTML toasts', function () {
     await this.browser.pause(PAUSE_TIME);
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Can be displayed', async function () {
     const equal = await this.browser.execute(() => {
       const element = document.createElement('div');
diff --git a/test/webdriverio/test/workspace_comment_test.ts b/test/webdriverio/test/workspace_comment_test.ts
@@ -20,6 +20,7 @@ import {
   keyUp,
   contextMenuItems,
   PAUSE_TIME,
+  checkForFailures,
 } from './test_setup.js';
 import {Key} from 'webdriverio';
 
@@ -69,6 +70,10 @@ suite('Workspace comment navigation', function () {
     };
   });
 
+  teardown(async function() {
+    await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state);
+  });
+
   test('Navigate forward from block to workspace comment', async function () {
     await focusOnBlock(this.browser, 'p5_canvas_1');
     await keyDown(this.browser);
