[Scout] Test config manifests

.buildkite/scripts/steps/checks/check_scout_config.sh

@@ -5,4 +5,11 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 
 echo --- Check for unregistered Scout Playwright configs
-node scripts/scout discover-playwright-configs --validate 
+node scripts/scout discover-playwright-configs --validate
+
+echo --- Make sure Scout config manifests are up to date
+node scripts/scout update-test-config-manifests
+check_for_changed_files \
+  "node scripts/scout update-test-config-manifests" \
+  true \
+  "[Scout] Automated config manifest updates"

.buildkite/scripts/steps/checks/quick_checks.json

@@ -75,7 +75,8 @@
     "mayChangeFiles": true
   },
   {
-    "script": ".buildkite/scripts/steps/checks/check_scout_config.sh"
+    "script": ".buildkite/scripts/steps/checks/check_scout_config.sh",
+    "mayChangeFiles": true
   },
   {
     "script": ".buildkite/scripts/steps/checks/dependencies_diff.sh",

src/platform/packages/private/kbn-scout-info/src/paths.ts

@@ -13,18 +13,41 @@ import { REPO_ROOT } from '@kbn/repo-info';
 export const SCOUT_OUTPUT_ROOT = path.resolve(REPO_ROOT, '.scout');
 
 // Servers
+
 export const SCOUT_SERVERS_ROOT = path.resolve(SCOUT_OUTPUT_ROOT, 'servers');
 
 // Reporting
+
 export const SCOUT_REPORT_OUTPUT_ROOT = path.resolve(SCOUT_OUTPUT_ROOT, 'reports');
 export const SCOUT_TEST_CONFIG_STATS_PATH = path.resolve(
   SCOUT_OUTPUT_ROOT,
   'test_config_stats.json'
 );
 
-// Scout playwright configs
+// Scout definitions
+
 export const SCOUT_PLAYWRIGHT_CONFIGS_PATH = path.resolve(
   SCOUT_OUTPUT_ROOT,
   'test_configs',
   'scout_playwright_configs.json'
 );
+
+export const TESTABLE_COMPONENT_SCOUT_ROOT_PATH_GLOB =
+  '{src/platform,x-pack/**}/{plugins,packages}/**/test/scout';
+
+export const TESTABLE_COMPONENT_SCOUT_ROOT_PATH_REGEX = new RegExp(
+  `(?:src|x-pack)` +
+    `\/(?:(platform)|solutions\/(\\w+))` + // 1: platform, 2: solution
+    `\/(plugins|packages)` + // 3: plugin or package
+    `\/?(shared|private|)` + // 4: artifact visibility
+    `\/([\\w|-]*)` + // 5: plugin/package name
+    `\/test\/scout(?:_([^\\/]*))?` // 6: custom target config set name
+);
+export const SCOUT_CONFIG_PATH_GLOB =
+  TESTABLE_COMPONENT_SCOUT_ROOT_PATH_GLOB + '/{ui,api}/{,*.}playwright.config.ts';
+
+export const SCOUT_CONFIG_PATH_REGEX = new RegExp(
+  TESTABLE_COMPONENT_SCOUT_ROOT_PATH_REGEX.source +
+    `\/(api|ui)` + // 7: Scout test category
+    `\/(\\w*)\\.?playwright.config.ts` // 8: Scout config type
+);

src/platform/packages/private/kbn-scout-reporting/src/helpers/index.ts

@@ -10,4 +10,4 @@
 export { getKibanaModuleData, type KibanaModuleMetadata } from './read_manifest';
 export { excapeHtmlCharacters, stripFilePath, parseStdout } from './text_processing';
 export { getRunTarget, stripRunCommand } from './cli_processing';
-export { getTestIDForTitle, generateTestRunId } from './test_id_generator';
+export { computeTestID, generateTestRunId } from './test_id_generator';

src/platform/packages/private/kbn-scout-reporting/src/helpers/test_id_generator.test.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { generateTestRunId, computeTestID } from './test_id_generator';
+import path from 'node:path';
+
+describe('generateTestRunId', () => {
+  it("generates random 16 char string every time it's called", () => {
+    const hashes: string[] = [];
+
+    for (let i = 0; i < 10; i++) {
+      const hash = generateTestRunId();
+      expect(hash).toHaveLength(16);
+      expect(hashes.includes(hash)).toBeFalsy();
+      hashes.push(hash);
+    }
+  });
+});
+
+describe('computeTestID', () => {
+  it('returns the same output every time if the inputs are the same', () => {
+    const getTestID = () => computeTestID(path.join('some_functionality.spec.ts'), test.name);
+    const expectedTestId = '5895f3c6f599ba8-9f86d081884c7d6'; // hard-coded to detect any changes in hash calculations
+    const testID = getTestID();
+
+    expect(testID).toEqual(expectedTestId);
+    expect(getTestID()).toEqual(testID);
+  });
+
+  it('output is two 15 char hashes joined by a dash', () => {
+    expect(computeTestID('foo/bar', 'baz bat')).toMatch(/\w{15}-\w{15}/);
+  });
+
+  it("doesn't accept zero-length inputs", () => {
+    expect(() => computeTestID('some/path', '')).toThrow();
+    expect(() => computeTestID('', 'some test title')).toThrow();
+  });
+});

src/platform/packages/private/kbn-scout-reporting/src/helpers/test_id_generator.ts

@@ -13,6 +13,15 @@ export function generateTestRunId() {
   return randomBytes(8).toString('hex');
 }
 
-export function getTestIDForTitle(title: string) {
-  return createHash('sha256').update(title).digest('hex').slice(0, 31);
+export function computeTestID(testFilePath: string, testTitle: string) {
+  if (testFilePath.length === 0 || testTitle.length === 0) {
+    throw new Error(
+      'Inputs used to compute test IDs cannot be zero-length' +
+        ` (got testFilePath='${testFilePath}', testTitle='${testTitle}')`
+    );
+  }
+
+  return [testFilePath, testTitle]
+    .map((input) => createHash('sha256').update(input).digest('hex').slice(0, 15))
+    .join('-');
 }

src/platform/packages/private/kbn-scout-reporting/src/registry/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export * from './test_config';
+export * from './testable_module';

src/platform/packages/private/kbn-scout-reporting/src/registry/manifest.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { simpleGit, type SimpleGit } from 'simple-git';
+import { REPO_ROOT } from '@kbn/repo-info';
+import type { TestCase } from '@playwright/test/reporter';
+
+let git: SimpleGit;
+
+export const getGitSHA1ForPath = async (p: string) => {
+  if (git === undefined) git = simpleGit(REPO_ROOT);
+  return (await git.raw(['ls-tree', '--object-only', 'HEAD', p])).trim();
+};
+
+export interface ScoutConfigManifest {
+  path: string;
+  exists: boolean;
+  lastModified: string;
+  sha1: string;
+  tests: {
+    id: string;
+    title: string;
+    expectedStatus: string;
+    tags: string[];
+    location: TestCase['location'];
+  }[];
+}