Support SNAT Fixed Port Ranges

Author: ethan-gallant

README.md

@@ -267,6 +267,16 @@ Default: empty
 Specify a comma-separated list of IPv4 CIDRs to exclude from SNAT. For every item in the list an `iptables` rule and off\-VPC
 IP rule will be applied. If an item is not a valid ipv4 range it will be skipped. This should be used when `AWS_VPC_K8S_CNI_EXTERNALSNAT=false`.
 
+#### `AWS_VPC_K8S_CNI_SNAT_FIXED_PORTS`
+
+Type: String
+
+Default: empty
+
+Specify a comma-separated list of ports or port ranges that should be excluded from port randomization when SNAT is applied. Format should be individual ports or port ranges, for example: "80,443,8080-8090". This takes effect when `AWS_VPC_K8S_CNI_EXTERNALSNAT=false` and `AWS_VPC_K8S_CNI_RANDOMIZESNAT` is set to either `hashrandom` or `prng`. The specified ports will still be SNATed but will maintain their original source port values instead of being randomized.
+
+*Note*: This is useful when you have applications that require consistent source ports for outbound connections, or when you need to ensure specific source ports are used for outbound traffic. The ports specified here will be excluded from the random port allocation mechanism while still being subject to SNAT rules.
+
 #### `POD_MTU` (v1.16.4+)
 
 Type: Integer as a String

cmd/aws-vpc-cni/main.go

@@ -104,6 +104,7 @@ const (
 	envEnIPv6Egress          = "ENABLE_V6_EGRESS"
 	envEnIPv4Egress          = "ENABLE_V4_EGRESS"
 	envRandomizeSNAT         = "AWS_VPC_K8S_CNI_RANDOMIZESNAT"
+	envSNATFixedPorts        = "AWS_VPC_K8S_CNI_SNAT_FIXED_PORTS"
 	envIPCooldownPeriod      = "IP_COOLDOWN_PERIOD"
 	envDisablePodV6          = "DISABLE_POD_V6"
 )
@@ -145,6 +146,8 @@ type NetConf struct {
 
 	RandomizeSNAT string `json:"randomizeSNAT,omitempty"`
 
+	SNATFixedPorts string `json:"snatFixedPorts,omitempty"`
+
 	// MTU for eth0
 	MTU string `json:"mtu,omitempty"`
 
@@ -266,6 +269,7 @@ func generateJSON(jsonFile string, outFile string, getPrimaryIP func(ipv4 bool)
 	pluginLogFile := utils.GetEnv(envPluginLogFile, defaultPluginLogFile)
 	pluginLogLevel := utils.GetEnv(envPluginLogLevel, defaultPluginLogLevel)
 	randomizeSNAT := utils.GetEnv(envRandomizeSNAT, defaultRandomizeSNAT)
+	snatFixedPorts := utils.GetEnv(envSNATFixedPorts, "")
 
 	netconf := string(byteValue)
 	netconf = strings.Replace(netconf, "__VETHPREFIX__", vethPrefix, -1)
@@ -279,6 +283,7 @@ func generateJSON(jsonFile string, outFile string, getPrimaryIP func(ipv4 bool)
 	netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMDST__", egressIPAMDst, -1)
 	netconf = strings.Replace(netconf, "__EGRESSPLUGINIPAMDATADIR__", egressIPAMDataDir, -1)
 	netconf = strings.Replace(netconf, "__RANDOMIZESNAT__", randomizeSNAT, -1)
+	netconf = strings.Replace(netconf, "__SNATFIXEDPORTS__", snatFixedPorts, -1)
 	netconf = strings.Replace(netconf, "__NODEIP__", nodeIP, -1)
 
 	byteValue = []byte(netconf)

cmd/egress-cni-plugin/egressContext.go

@@ -267,7 +267,8 @@ func (ec *egressContext) cmdAddEgressV4() (err error) {
 		for _, ipc := range ec.TmpResult.IPs {
 			if ipc.Address.IP.To4() != nil {
 				// add SNAT chain/rules necessary for the container IPv6 egress traffic
-				if err = snat.Add(ec.IPTablesIface, ec.NetConf.NodeIP, ipc.Address.IP, ipv4MulticastRange, ec.SnatChain, ec.SnatComment, ec.NetConf.RandomizeSNAT); err != nil {
+				if err = snat.Add(ec.IPTablesIface, ec.NetConf.NodeIP, ipc.Address.IP, ipv4MulticastRange, ec.SnatChain, ec.SnatComment,
+					ec.NetConf.RandomizeSNAT, ec.NetConf.SNATFixedPorts); err != nil {
 					return err
 				}
 			}
@@ -392,7 +393,14 @@ func (ec *egressContext) cmdAddEgressV6() (err error) {
 
 	// set up SNAT in host for container IPv6 egress traffic
 	// following line adds an ip6tables entries to NAT for IPv6 traffic between container v6if0 and node primary ENI (eth0)
-	err = snat.Add(ec.IPTablesIface, ec.NetConf.NodeIP, containerIPv6, ipv6MulticastRange, ec.SnatChain, ec.SnatComment, ec.NetConf.RandomizeSNAT)
+	err = snat.Add(ec.IPTablesIface,
+		ec.NetConf.NodeIP,
+		containerIPv6,
+		ipv6MulticastRange,
+		ec.SnatChain,
+		ec.SnatComment,
+		ec.NetConf.RandomizeSNAT,
+		ec.NetConf.SNATFixedPorts)
 	if err != nil {
 		ec.Log.Errorf("setup host snat failed: %v", err)
 		return err

cmd/egress-cni-plugin/netconf.go

@@ -46,6 +46,8 @@ type NetConf struct {
 
 	RandomizeSNAT string `json:"randomizeSNAT"`
 
+	SNATFixedPorts string `json:"snatFixedPorts"`
+
 	// IP to use as SNAT target
 	NodeIP net.IP `json:"nodeIP"`
 

cmd/egress-cni-plugin/snat/snat.go

@@ -14,19 +14,36 @@
 package snat
 
 import (
+	"fmt"
 	"net"
 
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/iptableswrapper"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/cniutils"
 )
 
-func iptRules(target, src net.IP, multicastRange, chain, comment string, useRandomFully, useHashRandom bool) [][]string {
+func iptRules(target, src net.IP, multicastRange, chain, comment string, useRandomFully, useHashRandom bool, fixedPorts string) [][]string {
 	var rules [][]string
 
 	// Accept/ignore multicast (just because we can)
 	rules = append(rules, []string{chain, "-d", multicastRange, "-j", "ACCEPT", "-m", "comment", "--comment", comment})
 
-	// SNAT
+	if fixedPorts != "" && (useRandomFully || useHashRandom) {
+		// Add protocol-specific SNAT rules for fixed ports
+		for _, proto := range []string{"tcp", "udp", "sctp", "dccp"} {
+			args := []string{
+				chain,
+				"-p", proto,
+				"-m", "multiport",
+				"--sports", fixedPorts,
+				"-j", "SNAT",
+				"--to-source", target.String(),
+				"-m", "comment", "--comment", fmt.Sprintf("%s (fixed ports %s)", comment, proto),
+			}
+			rules = append(rules, args)
+		}
+	}
+
+	// SNAT rule for remaining traffic (protocol-agnostic)
 	args := []string{
 		chain,
 		"-j", "SNAT",
@@ -47,7 +64,7 @@ func iptRules(target, src net.IP, multicastRange, chain, comment string, useRand
 }
 
 // Add NAT entries to iptables for POD egress IPv6/IPv4 traffic
-func Add(ipt iptableswrapper.IPTablesIface, nodeIP, src net.IP, multicastRange, chain, comment, rndSNAT string) error {
+func Add(ipt iptableswrapper.IPTablesIface, nodeIP, src net.IP, multicastRange, chain, comment, rndSNAT, fixedPorts string) error {
 	//Defaults to `random-fully` unless a different option is explicitly set via
 	//`AWS_VPC_K8S_CNI_RANDOMIZESNAT`. If the underlying iptables version doesn't support
 	//'random-fully`, we will fall back to `random`.
@@ -58,7 +75,7 @@ func Add(ipt iptableswrapper.IPTablesIface, nodeIP, src net.IP, multicastRange,
 		useHashRandom, useRandomFully = true, false
 	}
 
-	rules := iptRules(nodeIP, src, multicastRange, chain, comment, useRandomFully, useHashRandom)
+	rules := iptRules(nodeIP, src, multicastRange, chain, comment, useRandomFully, useHashRandom, fixedPorts)
 
 	chains, err := ipt.ListChains("nat")
 	if err != nil {

cmd/egress-cni-plugin/snat/snat_test.go

@@ -57,14 +57,52 @@ func TestAddV4(t *testing.T) {
 
 	setupAddExpect(ipt, &actualChain, &actualRule)
 
-	err := Add(ipt, nodeIPv4, containerIPv4, ipv4MulticastRange, chainV4, comment, rndSNAT)
+	err := Add(ipt, nodeIPv4, containerIPv4, ipv4MulticastRange, chainV4, comment, rndSNAT, "")
 	assert.Nil(t, err)
 
 	assert.EqualValuesf(t, expectChain, actualChain, "iptables chain is expected to be created")
 
 	assert.EqualValuesf(t, expectRule, actualRule, "iptables rules are expected to be created")
 }
 
+func TestAddV4WithExcludedPorts(t *testing.T) {
+	ipt := mock_iptables.NewMockIPTablesIface(gomock.NewController(t))
+
+	expectChain := []string{chainV4}
+	actualChain := []string{}
+
+	excludedPorts := "1024:2048,8080:8090"
+
+	expectRule := []string{
+		fmt.Sprintf("nat %s -d %s -j ACCEPT -m comment --comment %s",
+			chainV4, ipv4MulticastRange, comment),
+	}
+
+	for _, proto := range []string{"tcp", "udp", "sctp", "dccp"} {
+		expectRule = append(expectRule,
+			fmt.Sprintf("nat %s -p %s -m multiport --sports %s -j SNAT --to-source %s -m comment --comment %s (fixed ports %s)",
+				chainV4, proto, excludedPorts, nodeIPv4.String(), comment, proto))
+	}
+
+	expectRule = append(expectRule,
+		fmt.Sprintf("nat %s -j SNAT --to-source %s -m comment --comment %s --random",
+			chainV4, nodeIPv4.String(), comment))
+
+	expectRule = append(expectRule,
+		fmt.Sprintf("nat POSTROUTING -s %s -j %s -m comment --comment %s",
+			containerIPv4.String(), chainV4, comment))
+
+	actualRule := []string{}
+
+	setupAddExpect(ipt, &actualChain, &actualRule)
+
+	err := Add(ipt, nodeIPv4, containerIPv4, ipv4MulticastRange, chainV4, comment, rndSNAT, excludedPorts)
+	assert.Nil(t, err)
+
+	assert.EqualValuesf(t, expectChain, actualChain, "iptables chain is expected to be created")
+	assert.EqualValuesf(t, expectRule, actualRule, "iptables rules are expected to be created")
+}
+
 func TestDelV4(t *testing.T) {
 	ipt := mock_iptables.NewMockIPTablesIface(gomock.NewController(t))
 

misc/10-aws.conflist

@@ -18,6 +18,7 @@
       "mtu": "__MTU__",
       "enabled": "__EGRESSPLUGINENABLED__",
       "randomizeSNAT": "__RANDOMIZESNAT__",
+      "snatFixedPorts": "__SNATFIXEDPORTS__",
       "nodeIP": "__NODEIP__",
       "ipam": {
          "type": "host-local",

pkg/networkutils/network.go

@@ -97,6 +97,12 @@ const (
 	// Default is "prng".
 	envRandomizeSNAT = "AWS_VPC_K8S_CNI_RANDOMIZESNAT"
 
+	// This environment is used to specify port ranges which will be excluded from SNAT randomization.
+	// This is useful if you still want the benefit of randomization but need to exclude certain ports
+	// for compatibility with protocols such as STUN and TURN.
+	// Defaults to empty.
+	envSNATFixedPorts = "AWS_VPC_K8S_CNI_SNAT_FIXED_PORTS"
+
 	// envNodePortSupport is the name of environment variable that configures whether we implement support for
 	// NodePorts on the primary ENI. This requires that we add additional iptables rules and loosen the kernel's
 	// RPF check as described below. Defaults to true.
@@ -166,6 +172,7 @@ type linuxNetwork struct {
 	useExternalSNAT        bool
 	ipv6EgressEnabled      bool
 	excludeSNATCIDRs       []string
+	snatFixedPorts         []string
 	externalServiceCIDRs   []string
 	typeOfSNAT             snatType
 	nodePortSupportEnabled bool
@@ -193,6 +200,7 @@ func New() NetworkAPIs {
 		useExternalSNAT:        useExternalSNAT(),
 		ipv6EgressEnabled:      ipV6EgressEnabled(),
 		excludeSNATCIDRs:       parseCIDRString(envExcludeSNATCIDRs),
+		snatFixedPorts:         parsePortRangeString(envSNATFixedPorts),
 		externalServiceCIDRs:   parseCIDRString(envExternalServiceCIDRs),
 		typeOfSNAT:             typeOfSNAT(),
 		nodePortSupportEnabled: nodePortSupportEnabled(),
@@ -516,11 +524,36 @@ func (n *linuxNetwork) buildIptablesSNATRules(vpcCIDRs []string, primaryAddr *ne
 			}})
 	}
 
+	if len(n.snatFixedPorts) > 0 {
+		for _, proto := range []string{"tcp", "udp", "sctp", "dccp"} {
+			fixedPortRule := []string{
+				"!", "-o", "vlan+",
+				"-p", proto,
+				"-m", "multiport",
+				"-m", "comment", "--comment", fmt.Sprintf("AWS, SNAT (fixed ports %s)", proto),
+				"-m", "addrtype", "!", "--dst-type", "LOCAL",
+				"--sports", strings.Join(n.snatFixedPorts, ","),
+				"-j", "SNAT", "--to-source", primaryAddr.String(),
+			}
+
+			iptableRules = append(iptableRules, iptablesRule{
+				name:        "SNAT rule for fixed ports",
+				shouldExist: !n.useExternalSNAT,
+				table:       "nat",
+				chain:       chain,
+				rule:        fixedPortRule,
+			})
+		}
+	}
+
 	// Prepare the Desired Rule for SNAT Rule for non-pod ENIs
-	snatRule := []string{"!", "-o", "vlan+",
+	snatRule := []string{
+		"!", "-o", "vlan+",
 		"-m", "comment", "--comment", "AWS, SNAT",
 		"-m", "addrtype", "!", "--dst-type", "LOCAL",
-		"-j", "SNAT", "--to-source", primaryAddr.String()}
+		"-j", "SNAT", "--to-source", primaryAddr.String(),
+	}
+
 	if n.typeOfSNAT == randomHashSNAT {
 		snatRule = append(snatRule, "--random")
 	}
@@ -852,6 +885,7 @@ func GetConfigForDebug() map[string]interface{} {
 		envVethPrefix:           getVethPrefixName(),
 		envNodePortSupport:      nodePortSupportEnabled(),
 		envRandomizeSNAT:        typeOfSNAT(),
+		envSNATFixedPorts:       parsePortRangeString(envSNATFixedPorts),
 	}
 }
 
@@ -888,6 +922,57 @@ func (n *linuxNetwork) GetExternalServiceCIDRs() []string {
 	return parseCIDRString(envExternalServiceCIDRs)
 }
 
+// parsePortRangeString parses port ranges from the env Port ranges can be
+// comma separated singles ports "80,8080" or ranges "8070-8080". Returns iptables
+// compatible format with : for ranges.
+func parsePortRangeString(envVar string) []string {
+	portRangeString := os.Getenv(envVar)
+	if portRangeString == "" {
+		return nil
+	}
+
+	var ports []string
+	for _, p := range strings.Split(portRangeString, ",") {
+		p = strings.TrimSpace(p)
+		if strings.Contains(p, "-") {
+			rangeParts := strings.Split(p, "-")
+			if len(rangeParts) != 2 {
+				log.Errorf("%v is not a valid port range", p)
+				continue
+			}
+
+			start := validatePort(rangeParts[0], p)
+			if start == -1 {
+				continue
+			}
+
+			end := validatePort(rangeParts[1], p)
+			if end == -1 || end < start {
+				log.Errorf("Invalid end port in range %v", p)
+				continue
+			}
+
+			ports = append(ports, fmt.Sprintf("%d:%d", start, end))
+		} else {
+			port := validatePort(p, p)
+			if port == -1 {
+				continue
+			}
+			ports = append(ports, strconv.Itoa(port))
+		}
+	}
+	return ports
+}
+
+func validatePort(portStr string, originalInput string) int {
+	port, err := strconv.Atoi(strings.TrimSpace(portStr))
+	if err != nil || port < 1 || port > 65535 {
+		log.Errorf("Invalid port in %v", originalInput)
+		return -1
+	}
+	return port
+}
+
 func parseCIDRString(envVar string) []string {
 	cidrString := os.Getenv(envVar)
 	if cidrString == "" {