Skip to content

Commit 62bc138

Browse files
bryantbiggsbryantbiggs
committed
fix: Updates from testing and validation
1 parent 401cbbe commit 62bc138

File tree

8 files changed

+123
-25
lines changed

8 files changed

+123
-25
lines changed

‎docs/UPGRADE-3.0.md‎

Lines changed: 84 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ If you find a bug, please open an issue with supporting configuration to reprodu
8888
- None
8989

9090
- `virtual_cluster` sub-module
91+
- None
9192

9293
6. Added outputs:
9394

@@ -98,24 +99,101 @@ If you find a bug, please open an issue with supporting configuration to reprodu
9899
- None
99100

100101
- `virtual_cluster` sub-module
102+
- None
103+
104+
## Upgrade Migration
105+
106+
### Before v2.x Example
101107

102-
## Upgrade Migrations
108+
```hcl
109+
module "emr" {
110+
source = "terraform-aws-modules/emr/aws"
111+
version = "~> 2.0"
112+
113+
# Only the affected attributes are shown
114+
115+
bootstrap_action = {
116+
example = {
117+
name = "Just an example",
118+
path = "file:/bin/echo",
119+
args = ["Hello World!"]
120+
}
121+
}
122+
}
123+
```
103124

104-
Not applicable - there aren't any structural changes other than the security group rule changes noted above. A diff of before vs after would look identical.
125+
### After v3.x Example
126+
127+
```hcl
128+
module "emr" {
129+
source = "terraform-aws-modules/emr/aws"
130+
version = "~> 2.0"
131+
132+
# Only the affected attributes are shown
133+
134+
# Copy and paste from output to maintain backwards compatibility
135+
# This was added by the AWS Redshift API and provider in v6.x
136+
os_release_label = "2023.9.20251014.0"
137+
138+
bootstrap_action = [
139+
{
140+
name = "Just an example",
141+
path = "file:/bin/echo",
142+
args = ["Hello World!"]
143+
}
144+
]
145+
}
146+
```
105147

106148
### State Changes
107149

108150
Due to the change from `aws_security_group_rule` to `aws_vpc_security_group_ingress_rule` and `aws_vpc_security_group_egress_rule`, the following reference state changes are required to maintain the current security group rules. (Note: these are different resources so they cannot be moved with `terraform mv ...`)
109151

152+
#### Instance Group
153+
110154
```sh
155+
# Master Security Group
156+
terraform state rm 'module.emr_instance_group.aws_security_group_rule.master["default"]'
157+
terraform state import 'module.emr_instance_group.aws_vpc_security_group_egress_rule.master["all-traffic"]' 'sg-xxx'
158+
159+
# Slave Security Group
111160
terraform state rm 'module.emr_instance_group.aws_security_group_rule.slave["default"]'
112-
terraform state import 'module.emr_instance_group.aws_vpc_security_group_egress_rule.this["default"]' 'sg-xxx'
161+
terraform state import 'module.emr_instance_group.aws_vpc_security_group_egress_rule.slave["all-traffic"]' 'sg-xxx'
113162

114-
terraform state rm 'module.emr_instance_group.aws_security_group_rule.master["default"]'
115-
terraform state import 'module.emr_instance_group.aws_vpc_security_group_egress_rule.this["default"]' 'sg-xxx'
163+
# Service Security Group
164+
terraform state rm 'module.emr_instance_group.aws_security_group_rule.service["master_9443_ingress"]'
165+
terraform state import 'module.emr_instance_group.aws_vpc_security_group_ingress_rule.service["master_9443"]' 'sg-xxx'
166+
167+
terraform state rm 'module.emr_instance_group.aws_security_group_rule.service["master_9443_egress"]'
168+
terraform state import 'module.emr_instance_group.aws_vpc_security_group_egress_rule.service["master_8443"]' 'sg-xxx'
169+
170+
terraform state rm 'module.emr_instance_group.aws_security_group_rule.service["core_task_8443_egress"]'
171+
terraform state import 'module.emr_instance_group.aws_vpc_security_group_egress_rule.service["core_task_8443"]' 'sg-xxx'
172+
```
173+
174+
#### Instance Fleet
175+
176+
```sh
177+
# Master Security Group
178+
terraform state rm 'module.emr_instance_fleet.aws_security_group_rule.master["default"]'
179+
terraform state import 'module.emr_instance_fleet.aws_vpc_security_group_egress_rule.master["all-traffic"]' 'sg-xxx'
180+
181+
# Slave Security Group
182+
terraform state rm 'module.emr_instance_fleet.aws_security_group_rule.slave["default"]'
183+
terraform state import 'module.emr_instance_fleet.aws_vpc_security_group_egress_rule.slave["all-traffic"]' 'sg-xxx'
184+
185+
# Service Security Group
186+
terraform state rm 'module.emr_instance_fleet.aws_security_group_rule.service["master_9443_ingress"]'
187+
terraform state import 'module.emr_instance_fleet.aws_vpc_security_group_ingress_rule.service["master_9443"]' 'sg-xxx'
188+
189+
terraform state rm 'module.emr_instance_fleet.aws_security_group_rule.service["master_9443_egress"]'
190+
terraform state import 'module.emr_instance_fleet.aws_vpc_security_group_egress_rule.service["master_8443"]' 'sg-xxx'
191+
192+
terraform state rm 'module.emr_instance_fleet.aws_security_group_rule.service["core_task_8443_egress"]'
193+
terraform state import 'module.emr_instance_fleet.aws_vpc_security_group_egress_rule.service["core_task_8443"]' 'sg-xxx'
116194
```
117195

118-
Serverless sub-module
196+
#### Serverless sub-module
119197

120198
```sh
121199
terraform state rm 'module.emr_serverless_spark.aws_security_group_rule.this["egress_all"]'

‎examples/private-cluster/main.tf‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ locals {
3333
module "emr_instance_fleet" {
3434
source = "../.."
3535

36-
# create = false
3736
name = "${local.name}-instance-fleet"
3837

3938
release_label_filters = {
@@ -43,7 +42,7 @@ module "emr_instance_fleet" {
4342
}
4443
applications = ["spark", "trino"]
4544
auto_termination_policy = {
46-
idle_timeout = 3600
45+
idle_timeout = 14400
4746
}
4847

4948
bootstrap_action = [
@@ -177,7 +176,7 @@ module "emr_instance_group" {
177176
}
178177
applications = ["spark", "trino"]
179178
auto_termination_policy = {
180-
idle_timeout = 3600
179+
idle_timeout = 14400
181180
}
182181

183182
bootstrap_action = [

‎examples/public-cluster/main.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ module "emr_instance_fleet" {
4040
}
4141
applications = ["spark", "trino"]
4242
auto_termination_policy = {
43-
idle_timeout = 3600
43+
idle_timeout = 14400
4444
}
4545

4646
bootstrap_action = [
@@ -173,7 +173,7 @@ module "emr_instance_group" {
173173
}
174174
applications = ["spark", "trino"]
175175
auto_termination_policy = {
176-
idle_timeout = 3600
176+
idle_timeout = 14400
177177
}
178178

179179
bootstrap_action = [

‎examples/serverless-cluster/main.tf‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,7 @@ module "emr_serverless_spark" {
3535

3636
release_label_filters = {
3737
emr7 = {
38-
prefix = "emr-7"
39-
application = "spark"
38+
prefix = "emr-7"
4039
}
4140
}
4241

@@ -91,8 +90,7 @@ module "emr_serverless_hive" {
9190

9291
release_label_filters = {
9392
emr7 = {
94-
prefix = "emr-7"
95-
application = "hive"
93+
prefix = "emr-7"
9694
}
9795
}
9896

‎examples/virtual-cluster/main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ module "vpc_endpoints" {
234234
security_group_rules = {
235235
ingress_https = {
236236
description = "HTTPS from private subnets"
237-
cidr_blocks = join(",", module.vpc.private_subnets_cidr_blocks)
237+
cidr_blocks = module.vpc.private_subnets_cidr_blocks
238238
}
239239
}
240240

‎main.tf‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -511,7 +511,7 @@ resource "aws_emr_security_configuration" "this" {
511511

512512
locals {
513513
# Autoscaling not supported when using instance fleets
514-
create_autoscaling_iam_role = var.create && var.create_autoscaling_iam_role && (var.core_instance_fleet != null || var.master_instance_fleet != null) == 0
514+
create_autoscaling_iam_role = var.create && var.create_autoscaling_iam_role && var.core_instance_fleet == null && var.master_instance_fleet == null
515515
autoscaling_iam_role_name = coalesce(var.autoscaling_iam_role_name, "${var.name}-autoscaling")
516516
}
517517

@@ -788,7 +788,7 @@ resource "aws_vpc_security_group_ingress_rule" "master" {
788788
tags = merge(
789789
var.tags,
790790
var.managed_security_group_tags,
791-
{ Name = try(coalesce(each.value.name, "${local.master_security_group_name}-${each.key}"), null) },
791+
{ Name = try(coalesce(each.value.name, "${local.master_security_group_name}-${each.key}")) },
792792
each.value.tags
793793
)
794794
to_port = try(coalesce(each.value.to_port, each.value.from_port), null)
@@ -810,7 +810,7 @@ resource "aws_vpc_security_group_egress_rule" "master" {
810810
tags = merge(
811811
var.tags,
812812
var.managed_security_group_tags,
813-
{ Name = try(coalesce(each.value.name, "${local.master_security_group_name}-${each.key}"), null) },
813+
{ Name = try(coalesce(each.value.name, "${local.master_security_group_name}-${each.key}")) },
814814
each.value.tags
815815
)
816816
to_port = each.value.to_port
@@ -864,7 +864,7 @@ resource "aws_vpc_security_group_ingress_rule" "slave" {
864864
tags = merge(
865865
var.tags,
866866
var.managed_security_group_tags,
867-
{ Name = try(coalesce(each.value.name, "${local.slave_security_group_name}-${each.key}"), null) },
867+
{ Name = try(coalesce(each.value.name, "${local.slave_security_group_name}-${each.key}")) },
868868
each.value.tags
869869
)
870870
to_port = try(coalesce(each.value.to_port, each.value.from_port), null)
@@ -886,7 +886,7 @@ resource "aws_vpc_security_group_egress_rule" "slave" {
886886
tags = merge(
887887
var.tags,
888888
var.managed_security_group_tags,
889-
{ Name = try(coalesce(each.value.name, "${local.slave_security_group_name}-${each.key}"), null) },
889+
{ Name = try(coalesce(each.value.name, "${local.slave_security_group_name}-${each.key}")) },
890890
each.value.tags
891891
)
892892
to_port = each.value.to_port
@@ -903,6 +903,7 @@ locals {
903903
service_security_group_ingress_rules = merge(
904904
{
905905
"master_9443" = {
906+
name = null
906907
cidr_ipv4 = null
907908
cidr_ipv6 = null
908909
description = "Master security group secure communication"
@@ -921,6 +922,7 @@ locals {
921922
service_security_group_egress_rules = merge(
922923
{
923924
"core_task_8443" = {
925+
name = null
924926
cidr_ipv4 = null
925927
cidr_ipv6 = null
926928
description = "Allow the cluster manager to communicate with the core and task nodes"
@@ -933,6 +935,7 @@ locals {
933935
to_port = 8443
934936
}
935937
"master_8443" = {
938+
name = null
936939
cidr_ipv4 = null
937940
cidr_ipv6 = null
938941
description = "Allow the cluster manager to communicate with the masterk nodes"
@@ -989,7 +992,7 @@ resource "aws_vpc_security_group_ingress_rule" "service" {
989992
tags = merge(
990993
var.tags,
991994
var.managed_security_group_tags,
992-
{ Name = try(coalesce(each.value.name, "${local.service_security_group_name}-${each.key}"), null) },
995+
{ Name = try(coalesce(each.value.name, "${local.service_security_group_name}-${each.key}")) },
993996
each.value.tags
994997
)
995998
to_port = try(coalesce(each.value.to_port, each.value.from_port), null)
@@ -1011,7 +1014,7 @@ resource "aws_vpc_security_group_egress_rule" "service" {
10111014
tags = merge(
10121015
var.tags,
10131016
var.managed_security_group_tags,
1014-
{ Name = try(coalesce(each.value.name, "${local.service_security_group_name}-${each.key}"), null) },
1017+
{ Name = try(coalesce(each.value.name, "${local.service_security_group_name}-${each.key}")) },
10151018
each.value.tags
10161019
)
10171020
to_port = each.value.to_port

‎migrations.tf‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
################################################################################
2+
# Migrations: v2.4.3 -> v3.0
3+
################################################################################
4+
5+
moved {
6+
from = aws_emr_instance_group.this["0"]
7+
to = aws_emr_instance_group.this[0]
8+
}
9+
10+
moved {
11+
from = aws_emr_instance_fleet.this["0"]
12+
to = aws_emr_instance_fleet.this[0]
13+
}
14+
15+
16+
# module.emr_instance_fleet.aws_iam_role_policy_attachment.autoscaling[0] will be created
17+
# module.emr_instance_fleet.aws_iam_role.autoscaling[0] will be created
18+
19+
# module.emr_instance_group.aws_iam_role_policy_attachment.autoscaling[0] will be destroyed
20+
# module.emr_instance_group.aws_iam_role.autoscaling[0] will be destroyed

‎modules/serverless/main.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ data "aws_emr_release_labels" "this" {
77
for_each = var.release_label_filters
88

99
content {
10-
application = try(coalesce(filters.value.application, var.type))
10+
application = filters.value.application
1111
prefix = filters.value.prefix
1212
}
1313
}
@@ -109,7 +109,7 @@ resource "aws_emrserverless_application" "this" {
109109
}
110110
}
111111

112-
release_label = try(coalesce(var.release_label, element(data.aws_emr_release_labels.this[0].release_labels, 0)), "")
112+
release_label = try(coalesce(var.release_label, element(data.aws_emr_release_labels.this[0].release_labels, 0)))
113113

114114
dynamic "monitoring_configuration" {
115115
for_each = var.monitoring_configuration != null ? [var.monitoring_configuration] : []

0 commit comments

Comments
 (0)