Merge branch 'main' into feat(init)/support-package-manager-option

Author: mrgrain

packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js

@@ -502,6 +502,17 @@ class DriftableStack extends cdk.Stack {
   }
 }
 
+class EarlyValidationStack extends cdk.Stack {
+  constructor(parent, id, props) {
+    super(parent, id, props);
+
+    new s3.Bucket(this, 'MyBucket', {
+      bucketName: process.env.BUCKET_NAME,
+      removalPolicy: cdk.RemovalPolicy.DESTROY,
+    });
+  }
+}
+
 class IamRolesStack extends cdk.Stack {
   constructor(parent, id, props) {
     super(parent, id, props);
@@ -971,6 +982,9 @@ switch (stackSet) {
     new MetadataStack(app, `${stackPrefix}-metadata`);
 
     new DriftableStack(app, `${stackPrefix}-driftable`);
+
+    new EarlyValidationStack(app, `${stackPrefix}-early-validation-stack1`);
+    new EarlyValidationStack(app, `${stackPrefix}-early-validation-stack2`);
     break;
 
   case 'stage-using-context':

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-early-validation.integtest.ts

@@ -0,0 +1,31 @@
+import { randomUUID } from 'node:crypto';
+import { integTest, withDefaultFixture } from '../../../lib';
+
+jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
+
+integTest(
+  'deploy - early validation error',
+  withDefaultFixture(async (fixture) => {
+    const bucketName = randomUUID();
+
+    // First, deploy a stack that creates a bucket with a custom name, which we expect to succeed
+    await fixture.cdkDeploy('early-validation-stack1', {
+      modEnv: {
+        BUCKET_NAME: bucketName,
+      },
+    });
+
+    // Then deploy a different instance of the stack, that creates another
+    // bucket with the same name, to induce an early validation error
+    const stdErr = await fixture.cdkDeploy('early-validation-stack2', {
+      modEnv: {
+        BUCKET_NAME: bucketName,
+      },
+      allowErrExit: true,
+    });
+
+    expect(stdErr).toContain(`Resource of type 'AWS::S3::Bucket' with identifier '${bucketName}' already exists`,
+    );
+  }),
+);
+

packages/@aws-cdk/toolkit-lib/lib/api/aws-auth/sdk.ts

@@ -100,9 +100,14 @@ import type {
   DescribeStackResourceDriftsCommandInput,
   ExecuteStackRefactorCommandInput,
   DescribeStackRefactorCommandInput,
-  CreateStackRefactorCommandOutput, ExecuteStackRefactorCommandOutput,
+  CreateStackRefactorCommandOutput,
+  ExecuteStackRefactorCommandOutput,
+  DescribeEventsCommandOutput,
+  DescribeEventsCommandInput,
 } from '@aws-sdk/client-cloudformation';
 import {
+  paginateDescribeEvents,
+
   paginateListStacks,
   CloudFormationClient,
   ContinueUpdateRollbackCommand,
@@ -113,6 +118,7 @@ import {
   DeleteChangeSetCommand,
   DeleteGeneratedTemplateCommand,
   DeleteStackCommand,
+  DescribeEventsCommand,
   DescribeChangeSetCommand,
   DescribeGeneratedTemplateCommand,
   DescribeResourceScanCommand,
@@ -141,6 +147,7 @@ import {
   waitUntilStackRefactorCreateComplete,
   waitUntilStackRefactorExecuteComplete,
 } from '@aws-sdk/client-cloudformation';
+import type { OperationEvent } from '@aws-sdk/client-cloudformation/dist-types/models/models_0';
 import type {
   FilterLogEventsCommandInput,
   FilterLogEventsCommandOutput,
@@ -434,6 +441,7 @@ export interface ICloudFormationClient {
   deleteChangeSet(input: DeleteChangeSetCommandInput): Promise<DeleteChangeSetCommandOutput>;
   deleteGeneratedTemplate(input: DeleteGeneratedTemplateCommandInput): Promise<DeleteGeneratedTemplateCommandOutput>;
   deleteStack(input: DeleteStackCommandInput): Promise<DeleteStackCommandOutput>;
+  describeEvents(input: DescribeEventsCommandInput): Promise<DescribeEventsCommandOutput>;
   describeChangeSet(input: DescribeChangeSetCommandInput): Promise<DescribeChangeSetCommandOutput>;
   describeGeneratedTemplate(
     input: DescribeGeneratedTemplateCommandInput,
@@ -468,6 +476,7 @@ export interface ICloudFormationClient {
   describeStackEvents(input: DescribeStackEventsCommandInput): Promise<DescribeStackEventsCommandOutput>;
   listStackResources(input: ListStackResourcesCommandInput): Promise<StackResourceSummary[]>;
   paginatedListStacks(input: ListStacksCommandInput): Promise<StackSummary[]>;
+  paginatedDescribeEvents(input: DescribeEventsCommandInput): Promise<OperationEvent[]>;
   createStackRefactor(input: CreateStackRefactorCommandInput): Promise<CreateStackRefactorCommandOutput>;
   executeStackRefactor(input: ExecuteStackRefactorCommandInput): Promise<ExecuteStackRefactorCommandOutput>;
   waitUntilStackRefactorCreateComplete(input: DescribeStackRefactorCommandInput): Promise<WaiterResult>;
@@ -710,6 +719,8 @@ export class SDK {
         client.send(new DetectStackDriftCommand(input)),
       detectStackResourceDrift: (input: DetectStackResourceDriftCommandInput): Promise<DetectStackResourceDriftCommandOutput> =>
         client.send(new DetectStackResourceDriftCommand(input)),
+      describeEvents: (input: DescribeEventsCommandInput): Promise<DescribeEventsCommandOutput> =>
+        client.send(new DescribeEventsCommand(input)),
       describeChangeSet: (input: DescribeChangeSetCommandInput): Promise<DescribeChangeSetCommandOutput> =>
         client.send(new DescribeChangeSetCommand(input)),
       describeGeneratedTemplate: (
@@ -775,6 +786,14 @@ export class SDK {
         }
         return stackResources;
       },
+      paginatedDescribeEvents: async (input: DescribeEventsCommandInput): Promise<OperationEvent[]> => {
+        const stackResources = Array<OperationEvent>();
+        const paginator = paginateDescribeEvents({ client }, input);
+        for await (const page of paginator) {
+          stackResources.push(...(page.OperationEvents || []));
+        }
+        return stackResources;
+      },
       createStackRefactor: (input: CreateStackRefactorCommandInput): Promise<CreateStackRefactorCommandOutput> => {
         return client.send(new CreateStackRefactorCommand(input));
       },

packages/@aws-cdk/toolkit-lib/lib/api/deployments/cfn-api.ts

@@ -21,6 +21,10 @@ import { CloudFormationStack, makeBodyParameter } from '../cloudformation';
 import type { IoHelper } from '../io/private';
 import type { ResourcesToImport } from '../resource-import';
 
+export interface ValidationReporter {
+  fetchDetails(changeSetName: string, stackName: string): Promise<string>;
+}
+
 /**
  * Describe a changeset in CloudFormation, regardless of its current state.
  *
@@ -103,7 +107,7 @@ export async function waitForChangeSet(
   ioHelper: IoHelper,
   stackName: string,
   changeSetName: string,
-  { fetchAll }: { fetchAll: boolean },
+  { fetchAll, validationReporter }: { fetchAll: boolean; validationReporter?: ValidationReporter },
 ): Promise<DescribeChangeSetCommandOutput> {
   await ioHelper.defaults.debug(format('Waiting for changeset %s on stack %s to finish creating...', changeSetName, stackName));
   const ret = await waitFor(async () => {
@@ -121,9 +125,20 @@ export async function waitForChangeSet(
       return description;
     }
 
+    const isEarlyValidationError = description.Status === ChangeSetStatus.FAILED &&
+      description.StatusReason?.includes('AWS::EarlyValidation');
+
+    if (isEarlyValidationError) {
+      const details = await validationReporter?.fetchDetails(changeSetName, stackName);
+      if (details) {
+        throw new ToolkitError(details);
+      }
+    }
     // eslint-disable-next-line @stylistic/max-len
     throw new ToolkitError(
-      `Failed to create ChangeSet ${changeSetName} on ${stackName}: ${description.Status || 'NO_STATUS'}, ${description.StatusReason || 'no reason provided'}`,
+      `Failed to create ChangeSet ${changeSetName} on ${stackName}: ${description.Status || 'NO_STATUS'}, ${
+        description.StatusReason || 'no reason provided'
+      }`,
     );
   });
 

packages/@aws-cdk/toolkit-lib/lib/api/deployments/deploy-stack.ts

@@ -33,11 +33,13 @@ import type { SDK, SdkProvider, ICloudFormationClient } from '../aws-auth/privat
 import type { TemplateBodyParameter } from '../cloudformation';
 import { makeBodyParameter, CfnEvaluationException, CloudFormationStack } from '../cloudformation';
 import type { EnvironmentResources, StringWithoutPlaceholders } from '../environment';
+import { EnvironmentResourcesRegistry } from '../environment';
 import { HotswapPropertyOverrides, HotswapMode, ICON, createHotswapPropertyOverrides } from '../hotswap/common';
 import { tryHotswapDeployment } from '../hotswap/hotswap-deployments';
 import type { IoHelper } from '../io/private';
 import type { ResourcesToImport } from '../resource-import';
 import { StackActivityMonitor } from '../stack-events';
+import { EarlyValidationReporter } from './early-validation';
 
 export interface DeployStackOptions {
   /**
@@ -511,8 +513,12 @@ class FullCloudFormationDeployment {
 
     await this.ioHelper.defaults.debug(format('Initiated creation of changeset: %s; waiting for it to finish creating...', changeSet.Id));
     // Fetching all pages if we'll execute, so we can have the correct change count when monitoring.
+    const environmentResourcesRegistry = new EnvironmentResourcesRegistry();
+    const envResources = environmentResourcesRegistry.for(this.options.resolvedEnvironment, this.options.sdk, this.ioHelper);
+    const validationReporter = new EarlyValidationReporter(this.options.sdk, envResources);
     return waitForChangeSet(this.cfn, this.ioHelper, this.stackName, changeSetName, {
       fetchAll: willExecute,
+      validationReporter,
     });
   }
 

packages/@aws-cdk/toolkit-lib/lib/api/deployments/early-validation.ts

@@ -0,0 +1,52 @@
+import type { OperationEvent } from '@aws-sdk/client-cloudformation';
+import type { ValidationReporter } from './cfn-api';
+import type { SDK } from '../aws-auth/sdk';
+import type { EnvironmentResources } from '../environment';
+
+/**
+ * A ValidationReporter that checks for early validation errors right after
+ * creating the change set. If any are found, it throws an error listing all validation failures.
+ * If the DescribeEvents API call fails (for example, due to insufficient permissions),
+ * it logs a warning instead.
+ */
+export class EarlyValidationReporter implements ValidationReporter {
+  constructor(private readonly sdk: SDK, private readonly envResources: EnvironmentResources) {
+  }
+
+  public async fetchDetails(changeSetName: string, stackName: string): Promise<string> {
+    let operationEvents: OperationEvent[] = [];
+    try {
+      operationEvents = await this.getFailedEvents(stackName, changeSetName);
+    } catch (error) {
+      let currentVersion: number | undefined = undefined;
+      try {
+        currentVersion = (await this.envResources.lookupToolkit()).version;
+      } catch (e) {
+      }
+
+      return `The template cannot be deployed because of early validation errors, but retrieving more details about those
+errors failed (${error}). Make sure you have permissions to call the DescribeEvents API, or re-bootstrap
+your environment with the latest version of the CLI (need at least version 30, current version ${currentVersion ?? 'unknown'}).`;
+    }
+
+    let message = `ChangeSet '${changeSetName}' on stack '${stackName}' failed early validation`;
+    if (operationEvents.length > 0) {
+      const failures = operationEvents
+        .map((event) => `  - ${event.ValidationStatusReason} (at ${event.ValidationPath})`)
+        .join('\n');
+
+      message += `:\n${failures}\n`;
+    }
+    return message;
+  }
+
+  private async getFailedEvents(stackName: string, changeSetName: string) {
+    return this.sdk.cloudFormation().paginatedDescribeEvents({
+      StackName: stackName,
+      ChangeSetName: changeSetName,
+      Filters: {
+        FailedEvents: true,
+      },
+    });
+  }
+}

packages/@aws-cdk/toolkit-lib/test/api/deployments/deploy-stack.test.ts

@@ -7,6 +7,7 @@ import {
   DeleteChangeSetCommand,
   DeleteStackCommand,
   DescribeChangeSetCommand,
+  DescribeEventsCommand,
   DescribeStacksCommand,
   ExecuteChangeSetCommand,
   type ExecuteChangeSetCommandInput,
@@ -763,6 +764,49 @@ test('deployStack reports no change if describeChangeSet returns specific error'
   expect(deployResult.type === 'did-deploy-stack' && deployResult.noOp).toEqual(true);
 });
 
+test('deployStack throws error in case of early validation failures', async () => {
+  mockCloudFormationClient.on(DescribeChangeSetCommand).resolvesOnce({
+    Status: ChangeSetStatus.FAILED,
+    StatusReason: '(AWS::EarlyValidation::SomeError). Blah blah blah.',
+  });
+
+  mockCloudFormationClient.on(DescribeEventsCommand).resolves({
+    OperationEvents: [
+      {
+        ValidationStatus: 'FAILED',
+        ValidationStatusReason: 'Resource already exists',
+        ValidationPath: 'Resources/MyResource',
+      },
+    ],
+  });
+
+  await expect(
+    testDeployStack({
+      ...standardDeployStackArguments(),
+    }),
+  ).rejects.toThrow(`ChangeSet 'cdk-deploy-change-set' on stack 'withouterrors' failed early validation:
+  - Resource already exists (at Resources/MyResource)`);
+});
+
+test('deployStack warns when it cannot get the events in case of early validation errors', async () => {
+  mockCloudFormationClient.on(DescribeChangeSetCommand).resolvesOnce({
+    Status: ChangeSetStatus.FAILED,
+    StatusReason: '(AWS::EarlyValidation::SomeError). Blah blah blah.',
+  });
+
+  mockCloudFormationClient.on(DescribeEventsCommand).rejectsOnce({
+    message: 'AccessDenied',
+  });
+
+  await expect(
+    testDeployStack({
+      ...standardDeployStackArguments(),
+    }),
+  ).rejects.toThrow(`The template cannot be deployed because of early validation errors, but retrieving more details about those
+errors failed (Error: AccessDenied). Make sure you have permissions to call the DescribeEvents API, or re-bootstrap
+your environment with the latest version of the CLI (need at least version 30, current version 0).`);
+});
+
 test('deploy not skipped if template did not change but one tag removed', async () => {
   // GIVEN
   mockCloudFormationClient.on(DescribeStacksCommand).resolves({

packages/@aws-cdk/toolkit-lib/test/api/deployments/early-validation.test.ts

@@ -0,0 +1,47 @@
+import { EarlyValidationReporter } from '../../../lib/api/deployments/early-validation';
+
+it('returns details when there are failed validation events', async () => {
+  const sdkMock = {
+    cloudFormation: jest.fn().mockReturnValue({
+      paginatedDescribeEvents: jest.fn().mockResolvedValue([
+        { ValidationStatusReason: 'Resource already exists', ValidationPath: 'Resources/MyResource' },
+      ]),
+    }),
+  };
+  const envResourcesMock = { lookupToolkit: jest.fn().mockResolvedValue({ version: 30 }) };
+  const reporter = new EarlyValidationReporter(sdkMock as any, envResourcesMock as any);
+
+  await expect(reporter.fetchDetails('test-change-set', 'test-stack')).resolves.toEqual(
+    "ChangeSet 'test-change-set' on stack 'test-stack' failed early validation:\n  - Resource already exists (at Resources/MyResource)\n",
+  );
+});
+
+it('returns a summary when there are no failed validation events', async () => {
+  const sdkMock = {
+    cloudFormation: jest.fn().mockReturnValue({
+      paginatedDescribeEvents: jest.fn().mockResolvedValue([]),
+    }),
+  };
+  const envResourcesMock = { lookupToolkit: jest.fn().mockResolvedValue({ version: 30 }) };
+  const reporter = new EarlyValidationReporter(sdkMock as any, envResourcesMock as any);
+
+  await expect(reporter.fetchDetails('test-change-set', 'test-stack')).resolves.toEqual(
+    "ChangeSet 'test-change-set' on stack 'test-stack' failed early validation",
+  );
+});
+
+it('returns an explanatory message when DescribeEvents API call fails', async () => {
+  const sdkMock = {
+    cloudFormation: jest.fn().mockReturnValue({
+      paginatedDescribeEvents: jest.fn().mockRejectedValue(new Error('AccessDenied')),
+    }),
+  };
+  const envResourcesMock = { lookupToolkit: jest.fn().mockResolvedValue({ version: 29 }) };
+  const reporter = new EarlyValidationReporter(sdkMock as any, envResourcesMock as any);
+
+  const result = await reporter.fetchDetails('test-change-set', 'test-stack');
+
+  expect(result).toContain('The template cannot be deployed because of early validation errors');
+  expect(result).toContain('AccessDenied');
+  expect(result).toContain('29');
+});

packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml

@@ -631,6 +631,7 @@ Resources:
                   - cloudformation:DeleteChangeSet
                   - cloudformation:DescribeChangeSet
                   - cloudformation:DescribeStacks
+                  - cloudformation:DescribeEvents
                   - cloudformation:ExecuteChangeSet
                   - cloudformation:CreateStack
                   - cloudformation:UpdateStack
@@ -814,7 +815,7 @@ Resources:
       Name:
         Fn::Sub: '/cdk-bootstrap/${Qualifier}/version'
       # Also update this value below (see comment there)
-      Value: '29'
+      Value: '30'
 Outputs:
   BucketName:
     Description: The name of the S3 bucket owned by the CDK toolkit stack
@@ -849,4 +850,4 @@ Outputs:
     # {Fn::GetAtt} on an SSM Parameter is eventually consistent, and can fail with "parameter
     # doesn't exist" even after just having been created. To reduce our deploy failure rate, we
     # duplicate the value here and use a build-time test to ensure the two values are the same.
-    Value: '29'
+    Value: '30'