Skip to content

Commit 1937c18

Browse files
YakDriverYakDriver
authored
Merge pull request #362 from hashicorp/b-cut-expired-creds-retry
Avoid retries on expired credentials
2 parents 87c729d + c4d0ab0 commit 1937c18

File tree

6 files changed

+432
-4
lines changed

6 files changed

+432
-4
lines changed

‎aws_config.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ func GetAwsConfig(ctx context.Context, c *Config) (context.Context, aws.Config,
9696

9797
if !c.SkipCredsValidation {
9898
if _, _, err := getAccountIDAndPartitionFromSTSGetCallerIdentity(baseCtx, stsClient(baseCtx, awsConfig, c)); err != nil {
99-
return ctx, awsConfig, fmt.Errorf("error validating provider credentials: %w", err)
99+
return ctx, awsConfig, fmt.Errorf("validating provider credentials: %w", err)
100100
}
101101
}
102102

‎aws_config_test.go‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"github.com/aws/aws-sdk-go-v2/config"
2323
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
2424
"github.com/aws/aws-sdk-go-v2/service/sts"
25+
"github.com/aws/smithy-go"
2526
"github.com/aws/smithy-go/middleware"
2627
smithyhttp "github.com/aws/smithy-go/transport/http"
2728
"github.com/google/go-cmp/cmp"
@@ -94,6 +95,52 @@ func TestGetAwsConfig(t *testing.T) {
9495
servicemocks.MockStsGetCallerIdentityValidEndpoint,
9596
},
9697
},
98+
{
99+
Config: &Config{
100+
AccessKey: servicemocks.MockStaticAccessKey,
101+
Region: "us-east-1",
102+
SecretKey: servicemocks.MockStaticSecretKey,
103+
MaxRetries: 100,
104+
},
105+
Description: "ExpiredToken",
106+
ExpectedRegion: "us-east-1",
107+
ExpectedError: func(err error) bool {
108+
return strings.Contains(err.Error(), "ExpiredToken")
109+
},
110+
MockStsEndpoints: []*servicemocks.MockEndpoint{
111+
servicemocks.MockStsGetCallerIdentityInvalidBodyExpiredToken,
112+
},
113+
},
114+
{
115+
Config: &Config{
116+
AccessKey: servicemocks.MockStaticAccessKey,
117+
Region: "us-east-1",
118+
SecretKey: servicemocks.MockStaticSecretKey,
119+
},
120+
Description: "ExpiredTokenException",
121+
ExpectedRegion: "us-east-1",
122+
ExpectedError: func(err error) bool {
123+
return strings.Contains(err.Error(), "ExpiredTokenException")
124+
},
125+
MockStsEndpoints: []*servicemocks.MockEndpoint{
126+
servicemocks.MockStsGetCallerIdentityInvalidBodyExpiredTokenException,
127+
},
128+
},
129+
{
130+
Config: &Config{
131+
AccessKey: servicemocks.MockStaticAccessKey,
132+
Region: "us-east-1",
133+
SecretKey: servicemocks.MockStaticSecretKey,
134+
},
135+
Description: "RequestExpired",
136+
ExpectedRegion: "us-east-1",
137+
ExpectedError: func(err error) bool {
138+
return strings.Contains(err.Error(), "RequestExpired")
139+
},
140+
MockStsEndpoints: []*servicemocks.MockEndpoint{
141+
servicemocks.MockStsGetCallerIdentityInvalidBodyRequestExpired,
142+
},
143+
},
97144
{
98145
Config: &Config{
99146
AccessKey: servicemocks.MockStaticAccessKey,
@@ -3043,6 +3090,76 @@ func TestRetryHandlers(t *testing.T) {
30433090
return results
30443091
}(),
30453092
},
3093+
"no retries for ExpiredToken": {
3094+
NextHandler: func() middleware.FinalizeHandler {
3095+
num := 0
3096+
reqsErrs := make([]error, 2)
3097+
for i := 0; i < 2; i++ {
3098+
reqsErrs[i] = &smithy.OperationError{
3099+
ServiceID: "STS",
3100+
OperationName: "GetCallerIdentity",
3101+
Err: &smithyhttp.ResponseError{
3102+
Response: &smithyhttp.Response{
3103+
Response: &http.Response{
3104+
StatusCode: 403,
3105+
},
3106+
},
3107+
Err: &smithy.GenericAPIError{
3108+
Code: "ExpiredToken",
3109+
Message: "The security token included in the request is expired",
3110+
},
3111+
},
3112+
}
3113+
}
3114+
return middleware.FinalizeHandlerFunc(func(ctx context.Context, in middleware.FinalizeInput) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
3115+
if num >= len(reqsErrs) {
3116+
err = fmt.Errorf("more requests than expected")
3117+
} else {
3118+
err = reqsErrs[num]
3119+
num++
3120+
}
3121+
return out, metadata, err
3122+
})
3123+
},
3124+
Err: &smithy.OperationError{
3125+
ServiceID: "STS",
3126+
OperationName: "GetCallerIdentity",
3127+
Err: &smithyhttp.ResponseError{
3128+
Response: &smithyhttp.Response{
3129+
Response: &http.Response{
3130+
StatusCode: 403,
3131+
},
3132+
},
3133+
Err: &smithy.GenericAPIError{
3134+
Code: "ExpiredToken",
3135+
Message: "The security token included in the request is expired",
3136+
},
3137+
},
3138+
},
3139+
ExpectResults: func() retry.AttemptResults {
3140+
results := retry.AttemptResults{
3141+
Results: make([]retry.AttemptResult, 1),
3142+
}
3143+
results.Results[0] = retry.AttemptResult{
3144+
Err: &smithy.OperationError{
3145+
ServiceID: "STS",
3146+
OperationName: "GetCallerIdentity",
3147+
Err: &smithyhttp.ResponseError{
3148+
Response: &smithyhttp.Response{
3149+
Response: &http.Response{
3150+
StatusCode: 403,
3151+
},
3152+
},
3153+
Err: &smithy.GenericAPIError{
3154+
Code: "ExpiredToken",
3155+
Message: "The security token included in the request is expired",
3156+
},
3157+
},
3158+
},
3159+
}
3160+
return results
3161+
}(),
3162+
},
30463163
"stops at maxRetries for other network errors": {
30473164
NextHandler: func() middleware.FinalizeHandler {
30483165
num := 0

‎awsauth_test.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,48 @@ func TestGetAccountIDAndPartitionFromSTSGetCallerIdentity(t *testing.T) {
320320
},
321321
ErrCount: 1,
322322
},
323+
{
324+
Description: "sts:GetCallerIdentity ExpiredToken with invalid JSON response",
325+
MockEndpoints: []*servicemocks.MockEndpoint{
326+
servicemocks.MockStsGetCallerIdentityInvalidBodyExpiredToken,
327+
},
328+
ErrCount: 1,
329+
},
330+
{
331+
Description: "sts:GetCallerIdentity ExpiredToken with valid JSON response",
332+
MockEndpoints: []*servicemocks.MockEndpoint{
333+
servicemocks.MockStsGetCallerIdentityValidBodyExpiredToken,
334+
},
335+
ErrCount: 1,
336+
},
337+
{
338+
Description: "sts:GetCallerIdentity ExpiredTokenException with invalid JSON response",
339+
MockEndpoints: []*servicemocks.MockEndpoint{
340+
servicemocks.MockStsGetCallerIdentityInvalidBodyExpiredTokenException,
341+
},
342+
ErrCount: 1,
343+
},
344+
{
345+
Description: "sts:GetCallerIdentity ExpiredTokenException with valid JSON response",
346+
MockEndpoints: []*servicemocks.MockEndpoint{
347+
servicemocks.MockStsGetCallerIdentityValidBodyExpiredTokenException,
348+
},
349+
ErrCount: 1,
350+
},
351+
{
352+
Description: "sts:GetCallerIdentity RequestExpired with invalid JSON response",
353+
MockEndpoints: []*servicemocks.MockEndpoint{
354+
servicemocks.MockStsGetCallerIdentityInvalidBodyRequestExpired,
355+
},
356+
ErrCount: 1,
357+
},
358+
{
359+
Description: "sts:GetCallerIdentity RequestExpired with valid JSON response",
360+
MockEndpoints: []*servicemocks.MockEndpoint{
361+
servicemocks.MockStsGetCallerIdentityValidBodyRequestExpired,
362+
},
363+
ErrCount: 1,
364+
},
323365
{
324366
Description: "sts:GetCallerIdentity success",
325367
MockEndpoints: []*servicemocks.MockEndpoint{

‎servicemocks/mock.go‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,72 @@ const (
113113
<Message>User: arn:aws:iam::123456789012:user/Bob is not authorized to perform: sts:GetCallerIdentity</Message>
114114
</Error>
115115
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
116+
</ErrorResponse>`
117+
// MockStsGetCallerIdentityValidResponseBodyExpiredToken uses code "ExpiredToken", seemingly the most common
118+
// code. Errors usually have an invalid body but this may be fixed at some point.
119+
MockStsGetCallerIdentityValidResponseBodyExpiredToken = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
120+
<Error>
121+
<Type>Sender</Type>
122+
<Code>ExpiredToken</Code>
123+
<Message>The security token included in the request is expired</Message>
124+
</Error>
125+
<ResponseMetadata>
126+
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
127+
</ResponseMetadata>
128+
</ErrorResponse>`
129+
// MockStsGetCallerIdentityInvalidResponseBodyExpiredToken uses code "ExpiredToken", seemingly the most common
130+
// code. Errors usually have an invalid body.
131+
MockStsGetCallerIdentityInvalidResponseBodyExpiredToken = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
132+
<Error>
133+
<Type>Sender</Type>
134+
<Code>ExpiredToken</Code>
135+
<Message>The security token included in the request is expired</Message>
136+
</Error>
137+
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
138+
</ErrorResponse>`
139+
// MockStsGetCallerIdentityValidResponseBodyExpiredTokenException uses code "ExpiredTokenException", a more rare code
140+
// but used at least by Fargate. Errors usually have an invalid body but this may change.
141+
MockStsGetCallerIdentityValidResponseBodyExpiredTokenException = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
142+
<Error>
143+
<Type>Sender</Type>
144+
<Code>ExpiredTokenException</Code>
145+
<Message>The security token included in the request is expired</Message>
146+
</Error>
147+
<ResponseMetadata>
148+
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
149+
</ResponseMetadata>
150+
</ErrorResponse>`
151+
// MockStsGetCallerIdentityInvalidResponseBodyExpiredTokenException uses code "ExpiredTokenException", a more rare code
152+
// but used at least by Fargate. Errors usually have an invalid body but this may change.
153+
MockStsGetCallerIdentityInvalidResponseBodyExpiredTokenException = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
154+
<Error>
155+
<Type>Sender</Type>
156+
<Code>ExpiredTokenException</Code>
157+
<Message>The security token included in the request is expired</Message>
158+
</Error>
159+
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
160+
</ErrorResponse>`
161+
// MockStsGetCallerIdentityValidResponseBodyRequestExpired uses code "RequestExpired", a code only used in EC2.
162+
// Errors usually have an invalid body but this may change.
163+
MockStsGetCallerIdentityValidResponseBodyRequestExpired = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
164+
<Error>
165+
<Type>Sender</Type>
166+
<Code>RequestExpired</Code>
167+
<Message>The security token included in the request is expired</Message>
168+
</Error>
169+
<ResponseMetadata>
170+
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
171+
</ResponseMetadata>
172+
</ErrorResponse>`
173+
// MockStsGetCallerIdentityInvalidResponseBodyRequestExpired uses code "RequestExpired", a code only used in EC2.
174+
// Errors usually have an invalid body but this may change.
175+
MockStsGetCallerIdentityInvalidResponseBodyRequestExpired = `<ErrorResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
176+
<Error>
177+
<Type>Sender</Type>
178+
<Code>RequestExpired</Code>
179+
<Message>The security token included in the request is expired</Message>
180+
</Error>
181+
<RequestId>01234567-89ab-cdef-0123-456789abcdef</RequestId>
116182
</ErrorResponse>`
117183
MockStsGetCallerIdentityPartition = `aws`
118184
MockStsGetCallerIdentityValidResponseBody = `<GetCallerIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
@@ -211,6 +277,96 @@ var (
211277
StatusCode: http.StatusForbidden,
212278
},
213279
}
280+
MockStsGetCallerIdentityInvalidBodyExpiredToken = &MockEndpoint{
281+
Request: &MockRequest{
282+
Body: url.Values{
283+
"Action": []string{"GetCallerIdentity"},
284+
"Version": []string{"2011-06-15"},
285+
}.Encode(),
286+
Method: http.MethodPost,
287+
Uri: "/",
288+
},
289+
Response: &MockResponse{
290+
Body: MockStsGetCallerIdentityInvalidResponseBodyExpiredToken,
291+
ContentType: "text/xml",
292+
StatusCode: http.StatusForbidden,
293+
},
294+
}
295+
MockStsGetCallerIdentityValidBodyExpiredToken = &MockEndpoint{
296+
Request: &MockRequest{
297+
Body: url.Values{
298+
"Action": []string{"GetCallerIdentity"},
299+
"Version": []string{"2011-06-15"},
300+
}.Encode(),
301+
Method: http.MethodPost,
302+
Uri: "/",
303+
},
304+
Response: &MockResponse{
305+
Body: MockStsGetCallerIdentityValidResponseBodyExpiredToken,
306+
ContentType: "text/xml",
307+
StatusCode: http.StatusForbidden,
308+
},
309+
}
310+
MockStsGetCallerIdentityInvalidBodyExpiredTokenException = &MockEndpoint{
311+
Request: &MockRequest{
312+
Body: url.Values{
313+
"Action": []string{"GetCallerIdentity"},
314+
"Version": []string{"2011-06-15"},
315+
}.Encode(),
316+
Method: http.MethodPost,
317+
Uri: "/",
318+
},
319+
Response: &MockResponse{
320+
Body: MockStsGetCallerIdentityInvalidResponseBodyExpiredTokenException,
321+
ContentType: "text/xml",
322+
StatusCode: http.StatusForbidden,
323+
},
324+
}
325+
MockStsGetCallerIdentityValidBodyExpiredTokenException = &MockEndpoint{
326+
Request: &MockRequest{
327+
Body: url.Values{
328+
"Action": []string{"GetCallerIdentity"},
329+
"Version": []string{"2011-06-15"},
330+
}.Encode(),
331+
Method: http.MethodPost,
332+
Uri: "/",
333+
},
334+
Response: &MockResponse{
335+
Body: MockStsGetCallerIdentityValidResponseBodyExpiredTokenException,
336+
ContentType: "text/xml",
337+
StatusCode: http.StatusForbidden,
338+
},
339+
}
340+
MockStsGetCallerIdentityInvalidBodyRequestExpired = &MockEndpoint{
341+
Request: &MockRequest{
342+
Body: url.Values{
343+
"Action": []string{"GetCallerIdentity"},
344+
"Version": []string{"2011-06-15"},
345+
}.Encode(),
346+
Method: http.MethodPost,
347+
Uri: "/",
348+
},
349+
Response: &MockResponse{
350+
Body: MockStsGetCallerIdentityInvalidResponseBodyRequestExpired,
351+
ContentType: "text/xml",
352+
StatusCode: http.StatusForbidden,
353+
},
354+
}
355+
MockStsGetCallerIdentityValidBodyRequestExpired = &MockEndpoint{
356+
Request: &MockRequest{
357+
Body: url.Values{
358+
"Action": []string{"GetCallerIdentity"},
359+
"Version": []string{"2011-06-15"},
360+
}.Encode(),
361+
Method: http.MethodPost,
362+
Uri: "/",
363+
},
364+
Response: &MockResponse{
365+
Body: MockStsGetCallerIdentityValidResponseBodyRequestExpired,
366+
ContentType: "text/xml",
367+
StatusCode: http.StatusForbidden,
368+
},
369+
}
214370
MockStsGetCallerIdentityValidEndpoint = &MockEndpoint{
215371
Request: &MockRequest{
216372
Body: url.Values{

‎v2/awsv1shim/session.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,9 +129,17 @@ func GetSession(ctx context.Context, awsC *awsv2.Config, c *awsbase.Config) (*se
129129
sess.Handlers.Retry.PushBack(func(r *request.Request) {
130130
logger := logging.RetrieveLogger(r.Context())
131131

132+
if r.IsErrorExpired() {
133+
logger.Warn(ctx, "Disabling retries after next request due to expired credentials", map[string]any{
134+
"error": r.Error,
135+
})
136+
r.Retryable = aws.Bool(false)
137+
}
138+
132139
if r.RetryCount < constants.MaxNetworkRetryCount {
133140
return
134141
}
142+
135143
// RequestError: send request failed
136144
// caused by: Post https://FQDN/: dial tcp: lookup FQDN: no such host
137145
if tfawserr.ErrMessageAndOrigErrContain(r.Error, request.ErrCodeRequestError, "send request failed", "no such host") {

0 commit comments

Comments
 (0)