Skip to content

Commit 80cb77f

Browse files
dmlemeshkodmlemeshko
authored
[9.1] [scout] failed test reporter integration (#239771) (#242433)
# Backport This will backport the following commits from `main` to `9.1`: - [[scout] failed test reporter integration (#239771)](#239771) <!--- Backport version: 10.1.0 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport) <!--BACKPORT [{"author":{"name":"Dzmitry Lemechko","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-11-10T13:31:13Z","message":"[scout] failed test reporter integration (#239771)\n\n## Summary\n\ncloses https://github.com/elastic/kibana/issues/241171\n\nThis PR extends existing `kbn/failed-test-reporter-cli` package to\ncreate/update GH issues for Scout test failures on upstream branches\nwith specific extras:\n\n- support **ndjson** input format:\n`.scout/reports/scout-playwright-test-failures-*/scout-failures-*.ndjson`\n- match existing GH issues for Scout test **by test name only**: don't\ncreate a new issue _for each target_ where the same test fails, but only\nadd a new comment with the specified target\n- use a different issue template for scout with extended details\n\nIssue example: https://github.com/elastic/kibana/issues/241472\n\nComments example: https://github.com/elastic/kibana/issues/239777\n\n---------\n\nCo-authored-by: Copilot <[email protected]>\nCo-authored-by: kibanamachine <[email protected]>","sha":"9c436b83b672f5c44f91ff313c10d46292cea239","branchLabelMapping":{"^v9.3.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport:version","test:scout","v9.3.0","v8.19.7","v9.1.7","v9.2.1"],"title":"[scout] failed test reporter integration","number":239771,"url":"https://github.com/elastic/kibana/pull/239771","mergeCommit":{"message":"[scout] failed test reporter integration (#239771)\n\n## Summary\n\ncloses https://github.com/elastic/kibana/issues/241171\n\nThis PR extends existing `kbn/failed-test-reporter-cli` package to\ncreate/update GH issues for Scout test failures on upstream branches\nwith specific extras:\n\n- support **ndjson** input format:\n`.scout/reports/scout-playwright-test-failures-*/scout-failures-*.ndjson`\n- match existing GH issues for Scout test **by test name only**: don't\ncreate a new issue _for each target_ where the same test fails, but only\nadd a new comment with the specified target\n- use a different issue template for scout with extended details\n\nIssue example: https://github.com/elastic/kibana/issues/241472\n\nComments example: https://github.com/elastic/kibana/issues/239777\n\n---------\n\nCo-authored-by: Copilot <[email protected]>\nCo-authored-by: kibanamachine <[email protected]>","sha":"9c436b83b672f5c44f91ff313c10d46292cea239"}},"sourceBranch":"main","suggestedTargetBranches":["8.19","9.1"],"targetPullRequestStates":[{"branch":"main","label":"v9.3.0","branchLabelMappingKey":"^v9.3.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/239771","number":239771,"mergeCommit":{"message":"[scout] failed test reporter integration (#239771)\n\n## Summary\n\ncloses https://github.com/elastic/kibana/issues/241171\n\nThis PR extends existing `kbn/failed-test-reporter-cli` package to\ncreate/update GH issues for Scout test failures on upstream branches\nwith specific extras:\n\n- support **ndjson** input format:\n`.scout/reports/scout-playwright-test-failures-*/scout-failures-*.ndjson`\n- match existing GH issues for Scout test **by test name only**: don't\ncreate a new issue _for each target_ where the same test fails, but only\nadd a new comment with the specified target\n- use a different issue template for scout with extended details\n\nIssue example: https://github.com/elastic/kibana/issues/241472\n\nComments example: https://github.com/elastic/kibana/issues/239777\n\n---------\n\nCo-authored-by: Copilot <[email protected]>\nCo-authored-by: kibanamachine <[email protected]>","sha":"9c436b83b672f5c44f91ff313c10d46292cea239"}},{"branch":"8.19","label":"v8.19.7","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"9.1","label":"v9.1.7","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"9.2","label":"v9.2.1","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"url":"https://github.com/elastic/kibana/pull/242419","number":242419,"state":"OPEN"}]}] BACKPORT-->
1 parent c3a8b5a commit 80cb77f

File tree

14 files changed

+1307
-169
lines changed

14 files changed

+1307
-169
lines changed

‎.buildkite/scripts/lifecycle/post_command.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ IS_TEST_EXECUTION_STEP="$(buildkite-agent meta-data get "${BUILDKITE_JOB_ID}_is_
1111
if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
1212
echo "--- Upload Artifacts"
1313
buildkite-agent artifact upload '.scout/reports/scout-playwright-test-failures-*/**/*'
14+
buildkite-agent artifact upload '.scout/reports/scout-playwright-test-failures-*/scout-failures-*.ndjson'
1415
buildkite-agent artifact upload 'target/junit/**/*'
1516
buildkite-agent artifact upload 'target/kibana-coverage/jest/**/*'
1617
buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*'
@@ -48,7 +49,9 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
4849
--no-github-update --no-index-errors
4950
else
5051
echo "--- Run Failed Test Reporter"
51-
node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml'
52+
node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" \
53+
'target/junit/**/*.xml' \
54+
'.scout/reports/scout-playwright-test-failures-*/scout-failures-*.ndjson'
5255
fi
5356
fi
5457

‎packages/kbn-failed-test-reporter-cli/failed_tests_reporter/existing_failed_test_issues.test.ts‎

Lines changed: 195 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,12 @@
77
* License v3.0 only", or the "Server Side Public License, v 1".
88
*/
99

10-
import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log';
1110
import { createStripAnsiSerializer } from '@kbn/jest-serializers';
11+
import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log';
1212

13+
import type { FailedTestIssue } from './existing_failed_test_issues';
14+
import { ExistingFailedTestIssues } from './existing_failed_test_issues';
1315
import type { TestFailure } from './get_failures';
14-
import { ExistingFailedTestIssues, FailedTestIssue } from './existing_failed_test_issues';
1516

1617
expect.addSnapshotSerializer(createStripAnsiSerializer());
1718

@@ -165,3 +166,195 @@ it('captures a list of failed test issue, loads the bodies for each issue, and o
165166
}
166167
`);
167168
});
169+
170+
describe('Scout failures', () => {
171+
it('detects Scout failures correctly', () => {
172+
const existing = new ExistingFailedTestIssues(log);
173+
174+
const scoutFailure: TestFailure & { id: string; target: string; location: string } = {
175+
...mockTestFailure,
176+
classname: 'scout suite',
177+
name: 'scout test',
178+
id: 'test-id-1',
179+
target: 'stateful',
180+
location: '/path/to/test.ts',
181+
};
182+
183+
const ftrFailure: TestFailure = {
184+
...mockTestFailure,
185+
classname: 'ftr suite',
186+
name: 'ftr test',
187+
};
188+
189+
expect(existing.isScoutFailure(scoutFailure)).toBe(true);
190+
expect(existing.isScoutFailure(ftrFailure)).toBe(false);
191+
});
192+
193+
it('matches Scout failures by name only, ignoring target differences', async () => {
194+
const existing = new ExistingFailedTestIssues(log);
195+
196+
Axios.request.mockImplementation(({ data }: any) => ({
197+
data: {
198+
existingIssues: data.failures
199+
.filter((t: any) => t.name === 'scout test name')
200+
.map(
201+
(t: any): FailedTestIssue => ({
202+
classname: t.classname || 'scout suite',
203+
name: t.name,
204+
github: {
205+
htmlUrl: 'htmlurl(scout test name)',
206+
nodeId: 'nodeid(scout test name)',
207+
number: 123,
208+
body: 'FAILURE: scout test name',
209+
},
210+
})
211+
),
212+
},
213+
}));
214+
215+
// First Scout failure with target chrome
216+
const scoutFailure1: TestFailure & { id: string; target: string; location: string } = {
217+
...mockTestFailure,
218+
classname: 'scout suite',
219+
name: 'scout test name',
220+
id: 'test-id-1',
221+
target: 'serverless=es',
222+
location: '/path/to/test.ts',
223+
};
224+
225+
// Second Scout failure with same name but different target
226+
const scoutFailure2: TestFailure & { id: string; target: string; location: string } = {
227+
...mockTestFailure,
228+
classname: 'scout suite',
229+
name: 'scout test name',
230+
id: 'test-id-2',
231+
target: 'stateful',
232+
location: '/path/to/test.ts',
233+
};
234+
235+
// Load the first failure
236+
await existing.loadForFailures([scoutFailure1]);
237+
238+
// Both failures should match the same issue (by name only)
239+
const issue1 = existing.getForFailure(scoutFailure1);
240+
const issue2 = existing.getForFailure(scoutFailure2);
241+
242+
expect(issue1).toBeDefined();
243+
expect(issue2).toBeDefined();
244+
expect(issue1?.name).toBe('scout test name');
245+
expect(issue2?.name).toBe('scout test name');
246+
expect(issue1).toEqual(issue2);
247+
});
248+
249+
it('correctly identifies seen Scout failures by name only', async () => {
250+
const existing = new ExistingFailedTestIssues(log);
251+
252+
Axios.request.mockImplementation(() => ({
253+
data: {
254+
existingIssues: [],
255+
},
256+
}));
257+
258+
// First Scout failure with target chrome
259+
const scoutFailure1: TestFailure & { id: string; target: string; location: string } = {
260+
...mockTestFailure,
261+
classname: 'scout suite',
262+
name: 'scout test name',
263+
id: 'test-id-1',
264+
target: 'serverless=es',
265+
location: '/path/to/test.ts',
266+
};
267+
268+
// Second Scout failure with same name but different target
269+
const scoutFailure2: TestFailure & { id: string; target: string; location: string } = {
270+
...mockTestFailure,
271+
classname: 'scout suite',
272+
name: 'scout test name',
273+
id: 'test-id-2',
274+
target: 'stateful',
275+
location: '/path/to/test.ts',
276+
};
277+
278+
// Load the first failure
279+
await existing.loadForFailures([scoutFailure1]);
280+
281+
// The second failure should be considered "seen" because it has the same name
282+
// This tests the isFailureSeen logic
283+
await existing.loadForFailures([scoutFailure1, scoutFailure2]);
284+
285+
// Should only make one API call (for the first failure, second is already seen)
286+
expect(Axios.request).toHaveBeenCalledTimes(1);
287+
});
288+
289+
it('distinguishes between Scout and FTR failures correctly', async () => {
290+
const existing = new ExistingFailedTestIssues(log);
291+
292+
Axios.request.mockImplementation(({ data }: any) => ({
293+
data: {
294+
existingIssues: data.failures.map(
295+
(t: any, i: number): FailedTestIssue => ({
296+
classname: t.classname,
297+
name: t.name,
298+
github: {
299+
htmlUrl: `htmlurl(${t.classname}/${t.name})`,
300+
nodeId: `nodeid(${t.classname}/${t.name})`,
301+
number: i + 1,
302+
body: `FAILURE: ${t.classname}/${t.name}`,
303+
},
304+
})
305+
),
306+
},
307+
}));
308+
309+
const scoutFailure: TestFailure & { id: string; target: string; location: string } = {
310+
...mockTestFailure,
311+
classname: 'scout suite',
312+
name: 'scout test',
313+
id: 'test-id-1',
314+
target: 'serverless=es',
315+
location: '/path/to/test.ts',
316+
};
317+
318+
const ftrFailure: TestFailure = {
319+
...mockTestFailure,
320+
classname: 'ftr suite',
321+
name: 'ftr test',
322+
};
323+
324+
// Load both failures
325+
await existing.loadForFailures([scoutFailure, ftrFailure]);
326+
327+
// Each should find its own issue
328+
const scoutIssue = existing.getForFailure(scoutFailure);
329+
const ftrIssue = existing.getForFailure(ftrFailure);
330+
331+
expect(scoutIssue).toBeDefined();
332+
expect(scoutIssue?.name).toBe('scout test');
333+
expect(ftrIssue).toBeDefined();
334+
expect(ftrIssue?.name).toBe('ftr test');
335+
});
336+
337+
it('returns undefined for Scout failures when no matching issue exists', async () => {
338+
const existing = new ExistingFailedTestIssues(log);
339+
340+
Axios.request.mockImplementation(() => ({
341+
data: {
342+
existingIssues: [],
343+
},
344+
}));
345+
346+
const scoutFailure: TestFailure & { id: string; target: string; location: string } = {
347+
...mockTestFailure,
348+
classname: 'scout suite',
349+
name: 'scout test',
350+
id: 'test-id-1',
351+
target: 'stateful',
352+
location: '/path/to/test.ts',
353+
};
354+
355+
await existing.loadForFailures([scoutFailure]);
356+
357+
const issue = existing.getForFailure(scoutFailure);
358+
expect(issue).toBeUndefined();
359+
});
360+
});

‎packages/kbn-failed-test-reporter-cli/failed_tests_reporter/existing_failed_test_issues.ts‎

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,12 @@
99

1010
import { setTimeout } from 'timers/promises';
1111

12-
import Axios from 'axios';
1312
import { isAxiosRequestError, isAxiosResponseError } from '@kbn/dev-utils';
14-
import { ToolingLog } from '@kbn/tooling-log';
13+
import type { ToolingLog } from '@kbn/tooling-log';
14+
import Axios from 'axios';
1515

16-
import { GithubIssueMini } from './github_api';
17-
import { TestFailure } from './get_failures';
16+
import type { TestFailure } from './get_failures';
17+
import type { GithubIssueMini } from './github_api';
1818

1919
export interface FailedTestIssue {
2020
classname: string;
@@ -84,12 +84,38 @@ export class ExistingFailedTestIssues {
8484
this.log.debug('loaded', this.results.size - initialResultSize, 'existing test issues');
8585
}
8686

87+
isScoutFailure(failure: TestFailure): boolean {
88+
return 'id' in failure && 'target' in failure && 'location' in failure;
89+
}
90+
8791
getForFailure(failure: TestFailure) {
92+
// Check if this is a Scout failure
93+
const isScout = this.isScoutFailure(failure);
94+
8895
for (const [f, issue] of this.results) {
89-
if (f.classname === failure.classname && f.name === failure.name) {
90-
return issue;
96+
if (!issue) {
97+
continue;
98+
}
99+
100+
// Verify both input and key are the same type (both Scout or both FTR)
101+
const isKeyScoutFailure = this.isScoutFailure(f);
102+
103+
if (isScout) {
104+
// For Scout failures, match by test name only (ignore target)
105+
// Both must be Scout failures and names must match
106+
if (isKeyScoutFailure && f.name === failure.name) {
107+
return issue;
108+
}
109+
} else {
110+
// For FTR failures, match by classname and name
111+
// Both must be FTR failures (not Scout) and classname+name must match
112+
if (!isKeyScoutFailure && f.classname === failure.classname && f.name === failure.name) {
113+
return issue;
114+
}
91115
}
92116
}
117+
118+
return undefined;
93119
}
94120

95121
addNewlyCreated(failure: TestFailure, newIssue: GithubIssueMini) {
@@ -148,9 +174,21 @@ export class ExistingFailedTestIssues {
148174
}
149175

150176
private isFailureSeen(failure: TestFailure) {
177+
// Check if this is a Scout failure
178+
const isScout = this.isScoutFailure(failure);
179+
151180
for (const seen of this.results.keys()) {
152-
if (seen.classname === failure.classname && seen.name === failure.name) {
153-
return true;
181+
if (isScout) {
182+
// For Scout failures, match by test name only (ignore target)
183+
const isExistingScoutFailure = this.isScoutFailure(seen);
184+
if (isExistingScoutFailure && seen.name === failure.name) {
185+
return true;
186+
}
187+
} else {
188+
// For FTR failures, use original matching logic
189+
if (seen.classname === failure.classname && seen.name === failure.name) {
190+
return true;
191+
}
154192
}
155193
}
156194

0 commit comments

Comments
 (0)