feat(integ-runner): add --strict flag to throw error on missing tests (#929)

## Description of the change
This PR adds a new flag `--strict` to integ-runner. When enabled, this
flag enforces that all specified tests are found and can be run.

With `strict` = `true`, integ-runner will throw an error when any
specified tests are not found.

## Reason for this change
For the [Integration Test
Workflow](https://github.com/aws/aws-cdk/blob/main/.github/workflows/integration-test-deployment.yml),
we want to throw an error if some specified tests are not found. This
ensures that all tests that have been changed in a PR are actually run
in the workflow.

Currently, integ-runner does not throw an error when tests are not
found. We can use the `--strict` flag in the workflow for our use case.

## Test
Added unit tests to verify the functionality of the new flag.

---
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license.

Author: abidhasan-aws

packages/@aws-cdk/integ-runner/README.md

@@ -73,6 +73,8 @@ If not, changes cannot be compared across systems and the [update workflow](#upd
   List of AWS Profiles to use when running tests in parallel
 - `--exclude` (default=`false`)
   If this is set to `true` then the list of tests provided will be excluded
+- `--strict` (default=`false`)
+  Fail if any specified tests are not found. Cannot be used with `--exclude`
 - `--from-file`
   Read the list of tests from this file
 - `--disable-update-workflow` (default=`false`)

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -37,6 +37,7 @@ export function parseCliArgs(args: string[] = []) {
     .options('profiles', { type: 'array', desc: 'list of AWS profiles to use. Tests will be run in parallel across each profile+regions', default: [] })
     .options('max-workers', { type: 'number', desc: 'The max number of workerpool workers to use when running integration tests in parallel', default: 16 })
     .options('exclude', { type: 'boolean', desc: 'Run all tests in the directory, except the specified TESTs', default: false })
+    .option('strict', { type: 'boolean', default: false, desc: 'Fail if any specified tests are not found' })
     .options('from-file', { type: 'string', desc: 'Read TEST names from a file (one TEST per line)' })
     .option('inspect-failures', { type: 'boolean', desc: 'Keep the integ test cloud assembly if a failure occurs for inspection', default: false })
     .option('disable-update-workflow', { type: 'boolean', default: false, desc: 'If this is "true" then the stack update workflow will be disabled' })
@@ -71,6 +72,11 @@ export function parseCliArgs(args: string[] = []) {
   if (tests.length > 0 && fromFile) {
     throw new Error('A list of tests cannot be provided if "--from-file" is provided');
   }
+
+  if (argv.strict && argv.exclude) {
+    throw new Error('Cannot use --strict with --exclude');
+  }
+
   const requestedTests = fromFile
     ? (fs.readFileSync(fromFile, { encoding: 'utf8' })).split('\n').filter(x => x)
     : (tests.length > 0 ? tests : undefined); // 'undefined' means no request
@@ -97,6 +103,7 @@ export function parseCliArgs(args: string[] = []) {
     disableUpdateWorkflow: argv['disable-update-workflow'] as boolean,
     language: arrayFromYargs(argv.language),
     watch: argv.watch as boolean,
+    strict: argv.strict as boolean,
     unstable: arrayFromYargs(argv.unstable) ?? [],
   };
 }

packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts

@@ -159,6 +159,13 @@ export interface IntegrationTestsDiscoveryOptions {
    */
   readonly exclude?: boolean;
 
+  /**
+   * If this is set to true, throw an error if any specified tests are not found
+   *
+   * @default false
+   */
+  readonly strict?: boolean;
+
   /**
    * List of tests to include (or exclude if `exclude=true`)
    *
@@ -204,10 +211,12 @@ export class IntegrationTests {
     language?: string[];
     testRegex?: string[];
     tests?: string[];
+    strict?: boolean;
   }): Promise<IntegTest[]> {
     const baseOptions = {
       tests: options.tests,
       exclude: options.exclude,
+      strict: options.strict,
     };
 
     // Explicitly set both, app and test-regex
@@ -283,7 +292,7 @@ export class IntegrationTests {
    *   If they have provided a test name that we don't find, then we write out that error message.
    * - If it is a list of tests to exclude, then we discover all available tests and filter out the tests that were provided by the user.
    */
-  private filterTests(discoveredTests: IntegTest[], requestedTests?: string[], exclude?: boolean): IntegTest[] {
+  private filterTests(discoveredTests: IntegTest[], requestedTests?: string[], exclude?: boolean, strict?: boolean): IntegTest[] {
     if (!requestedTests) {
       return discoveredTests;
     }
@@ -301,6 +310,9 @@ export class IntegrationTests {
       }
       if (unmatchedPatterns.length > 0) {
         process.stderr.write(`Available tests: ${discoveredTests.map(t => t.discoveryRelativeFileName).join(' ')}\n`);
+        if (strict) {
+          throw new Error(`Strict mode: ${unmatchedPatterns.length} test(s) not found: ${unmatchedPatterns.join(', ')}`);
+        }
         return [];
       }
     }
@@ -333,7 +345,7 @@ export class IntegrationTests {
 
     const discoveredTests = ignoreUncompiledTypeScript ? this.filterUncompiledTypeScript(testCases) : testCases;
 
-    return this.filterTests(discoveredTests, options.tests, options.exclude);
+    return this.filterTests(discoveredTests, options.tests, options.exclude, options.strict);
   }
 
   private filterUncompiledTypeScript(testCases: IntegTest[]): IntegTest[] {

packages/@aws-cdk/integ-runner/test/cli.test.ts

@@ -125,6 +125,15 @@ describe.each([
       '--app="node --prof {filePath}"',
     ])).rejects.toThrow('Only a single "--language" can be used with "--app". Alternatively provide both "--app" and "--test-regex" to fully customize the configuration.');
   });
+
+  test('cannot use --strict with --exclude', async () => {
+    await expect(() => cli([
+      'xxxxx.integ-test1.js',
+      '--language=javascript',
+      '--strict',
+      '--exclude',
+    ])).rejects.toThrow('Cannot use --strict with --exclude');
+  });
 });
 
 describe('CLI config file', () => {

packages/@aws-cdk/integ-runner/test/runner/integration-tests.test.ts

@@ -82,6 +82,14 @@ describe('IntegrationTests Discovery', () => {
         );
       });
 
+      test('test not found throws error only in Strict mode', async () => {
+        const testNames = [`test-data/${namedTest}`, `test-data/${namedTest.replace('test1', 'nonexistent')}`];
+
+        await expect(() => tests.fromCliOptions({ ...cliOptions, tests: testNames })).not.toThrow();
+        await expect(() => tests.fromCliOptions({ ...cliOptions, tests: testNames, strict: true }))
+          .rejects.toThrow(`Strict mode: 1 test(s) not found: test-data/${namedTest.replace('test1', 'nonexistent')}`);
+      });
+
       test('exclude tests', async () => {
         const integTests = await tests.fromCliOptions({ ...cliOptions, tests: [`test-data/${namedTest}`], exclude: true });