Skip to content

Commit 9f029ee

Browse files
m00leculem00lecule
committed
feat: introduce policy event logs scope
1 parent dbd95be commit 9f029ee

File tree

10 files changed

+137
-31
lines changed

10 files changed

+137
-31
lines changed

‎main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ func main() {
114114

115115
npMode, isMultiNICEnabled := lo.Must2(getNetworkPolicyConfigsFromIpamd(log))
116116

117-
ebpfClient := lo.Must1(ebpf.NewBpfClient(nodeIP, ctrlConfig.EnablePolicyEventLogs, ctrlConfig.EnableCloudWatchLogs,
117+
ebpfClient := lo.Must1(ebpf.NewBpfClient(nodeIP, ctrlConfig.EnablePolicyEventLogs, ctrlConfig.PolicyEventLogsScope, ctrlConfig.EnableCloudWatchLogs,
118118
ctrlConfig.EnableIPv6, ctrlConfig.ConntrackCacheCleanupPeriod, ctrlConfig.ConntrackCacheTableSize, npMode, isMultiNICEnabled))
119119
ebpfClient.ReAttachEbpfProbes()
120120

‎pkg/config/controller_config.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ const (
2121
defaultMaxConcurrentReconciles = 3
2222
defaultConntrackCacheCleanupPeriod = 300
2323
defaultConntrackCacheTableSize = 512 * 1024
24+
defaultPolicyEventLogsScope = "ACCEPT"
2425
flagEnablePolicyEventLogs = "enable-policy-event-logs"
26+
flagPolicyEventLogsScope = "policy-event-logs-scope"
2527
flagEnableCloudWatchLogs = "enable-cloudwatch-logs"
2628
flagEnableIPv6 = "enable-ipv6"
2729
flagEnableNetworkPolicy = "enable-network-policy"
@@ -58,6 +60,8 @@ type ControllerConfig struct {
5860
RuntimeConfig RuntimeConfig
5961
// Configuration for enabling profiling
6062
EnableProfiling bool
63+
// Policy event logs scope
64+
PolicyEventLogsScope string
6165
}
6266

6367
func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) {
@@ -80,7 +84,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) {
8084
"Cleanup interval for network policy agent conntrack cache")
8185
fs.IntVar(&cfg.ConntrackCacheTableSize, flagConntrackCacheTableSize, defaultConntrackCacheTableSize, ""+
8286
"Table size for network policy agent conntrack cache")
83-
87+
fs.StringVar(&cfg.PolicyEventLogsScope, flagPolicyEventLogsScope, defaultPolicyEventLogsScope, ""+
88+
"Set the policy event logs scope, if set to ACCEPT both ACCEPT and DENY events are generated, if set to DENY only DENY events are generated - ACCEPT, DENY")
8489
cfg.RuntimeConfig.BindFlags(fs)
8590
}
8691

‎pkg/ebpf/bpf_client.go‎

Lines changed: 56 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -44,19 +44,22 @@ var (
4444
TC_EGRESS_POD_STATE_MAP = "egress_pod_state_map"
4545
AWS_CONNTRACK_MAP = "aws_conntrack_map"
4646
AWS_EVENTS_MAP = "policy_events"
47+
AWS_EVENTS_SCOPE_MAP = "policy_events_scope"
4748
EKS_CLI_BINARY = "aws-eks-na-cli"
4849
EKS_V6_CLI_BINARY = "aws-eks-na-cli-v6"
4950
hostBinaryPath = "/host/opt/cni/bin/"
5051
IPv4_HOST_MASK = "/32"
5152
IPv6_HOST_MASK = "/128"
5253
CONNTRACK_MAP_PIN_PATH = "/sys/fs/bpf/globals/aws/maps/global_aws_conntrack_map"
5354
POLICY_EVENTS_MAP_PIN_PATH = "/sys/fs/bpf/globals/aws/maps/global_policy_events"
55+
POLICY_EVENTS_SCOPE_MAP_PIN_PATH = "/sys/fs/bpf/globals/aws/maps/global_policy_events_scope"
5456
CATCH_ALL_PROTOCOL corev1.Protocol = "ANY_IP_PROTOCOL"
5557
POD_VETH_PREFIX = "eni"
5658
POLICIES_APPLIED = 0
5759
DEFAULT_ALLOW = 1
5860
DEFAULT_DENY = 2
5961
POD_STATE_MAP_KEY = 0
62+
POLICY_EVENTS_SCOPE_MAP_KEY = 0
6063
BRANCH_ENI_VETH_PREFIX = "vlan"
6164
INTERFACE_COUNT_UNKNOWN = -1 // Used when caller doesn't know interface count
6265
INTERFACE_COUNT_DEFAULT = 1 // Default single interface
@@ -90,6 +93,10 @@ type pod_state struct {
9093
state uint8
9194
}
9295

96+
type policy_scope struct {
97+
scope uint8
98+
}
99+
93100
func msSince(start time.Time) float64 {
94101
return float64(time.Since(start) / time.Millisecond)
95102
}
@@ -118,9 +125,10 @@ type BPFContext struct {
118125
conntrackMapInfo goebpfmaps.BpfMap
119126
}
120127

121-
func NewBpfClient(nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs bool,
128+
func NewBpfClient(nodeIP string, enablePolicyEventLogs bool, policyEventsLogsScope string, enableCloudWatchLogs bool,
122129
enableIPv6 bool, conntrackTTL int, conntrackTableSize int, networkPolicyMode string, isMultiNICEnabled bool) (*bpfClient, error) {
123130
var conntrackMap goebpfmaps.BpfMap
131+
var policyEventsScopeMap goebpfmaps.BpfMap
124132

125133
ebpfClient := &bpfClient{
126134
// Maps PolicyEndpoint resource to it's eBPF context
@@ -147,7 +155,7 @@ func NewBpfClient(nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs boo
147155
ebpfClient.hostMask = ingressBinary, egressBinary, hostMask
148156

149157
bpfBinaries := []string{eventsBinary, ingressBinary, egressBinary, cliBinary}
150-
isConntrackMapPresent, isPolicyEventsMapPresent := false, false
158+
isConntrackMapPresent, isPolicyEventsMapPresent, isPolicyEventsScopeMapPresent := false, false, false
151159
var err error
152160

153161
ebpfClient.bpfSDKClient = goelf.New()
@@ -182,7 +190,7 @@ func NewBpfClient(nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs boo
182190
var interfaceNametoIngressPinPath map[string]string
183191
var interfaceNametoEgressPinPath map[string]string
184192
eventBufferFD := 0
185-
isConntrackMapPresent, isPolicyEventsMapPresent, eventBufferFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, err = ebpfClient.recoverBPFState(ebpfClient.bpfTCClient, ebpfClient.bpfSDKClient, ebpfClient.policyEndpointeBPFContext,
193+
isConntrackMapPresent, isPolicyEventsMapPresent, isPolicyEventsScopeMapPresent, eventBufferFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, err = ebpfClient.recoverBPFState(ebpfClient.bpfTCClient, ebpfClient.bpfSDKClient, ebpfClient.policyEndpointeBPFContext,
186194
ebpfClient.globalMaps, ingressUpdateRequired, egressUpdateRequired, eventsUpdateRequired)
187195
if err != nil {
188196
//Log the error and move on
@@ -197,7 +205,7 @@ func NewBpfClient(nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs boo
197205
// - Current events binary packaged with network policy agent is different than the one installed
198206
// during the previous installation (or)
199207
// - Either Conntrack Map (or) Events Map is currently missing on the node
200-
if eventsUpdateRequired || (!isConntrackMapPresent || !isPolicyEventsMapPresent) {
208+
if eventsUpdateRequired || (!isConntrackMapPresent || !isPolicyEventsMapPresent || !isPolicyEventsScopeMapPresent) {
201209
log().Info("Install the default global maps")
202210
eventsProbe := EVENTS_BINARY
203211
if enableIPv6 {
@@ -227,6 +235,10 @@ func NewBpfClient(nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs boo
227235
if mapName == AWS_EVENTS_MAP {
228236
eventBufferFD = int(mapInfo.MapFD)
229237
}
238+
if mapName == AWS_EVENTS_SCOPE_MAP {
239+
policyEventsScopeMap = mapInfo
240+
isPolicyEventsScopeMapPresent = true
241+
}
230242
}
231243
}
232244

@@ -245,6 +257,36 @@ func NewBpfClient(nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs boo
245257
ebpfClient.conntrackClient = conntrack.NewConntrackClient(conntrackMap, enableIPv6)
246258
log().Info("Initialized Conntrack client")
247259

260+
//if present update the PolicyEventsScope Map
261+
if isPolicyEventsScopeMapPresent {
262+
recoveredPolicyEventsScopeMap, ok := ebpfClient.globalMaps.Load(POLICY_EVENTS_SCOPE_MAP_PIN_PATH)
263+
if ok {
264+
policyEventsScopeMap = recoveredPolicyEventsScopeMap.(goebpfmaps.BpfMap)
265+
log().Info("Derived existing policyEventsScopeMap identifier")
266+
} else {
267+
log().Errorf("Unable to get policyEventsScopeMap post recovery..error: %v", err)
268+
sdkAPIErr.WithLabelValues("RecoveryFailed").Inc()
269+
return nil, err
270+
}
271+
272+
key := uint32(POLICY_EVENTS_SCOPE_MAP_KEY)
273+
scope := uint8(utils.ACCEPT.Index())
274+
275+
if policyEventsLogsScope == "DENY" {
276+
scope = uint8(utils.DENY.Index())
277+
}
278+
279+
value := policy_scope{scope: scope}
280+
log().Infof("Will update Policy Events Scope Map: key=%d value=%v", key, value)
281+
err := policyEventsScopeMap.CreateUpdateMapEntry(uintptr(unsafe.Pointer(&key)), uintptr(unsafe.Pointer(&value)), 0)
282+
283+
if err != nil {
284+
log().Errorf("Policy Events Scope Map update failed: %v", err)
285+
sdkAPIErr.WithLabelValues("updateEbpfMap-policy-events-scope").Inc()
286+
}
287+
log().Infof("Updated Policy Events Scope Map: key=%d value=%v", key, value)
288+
}
289+
248290
if enablePolicyEventLogs {
249291
err = events.ConfigurePolicyEventsLogging(enableCloudWatchLogs, eventBufferFD, enableIPv6)
250292
if err != nil {
@@ -376,8 +418,8 @@ func checkAndUpdateBPFBinaries(bpfTCClient tc.BpfTc, bpfBinaries []string, hostB
376418
}
377419

378420
func (l *bpfClient) recoverBPFState(bpfTCClient tc.BpfTc, eBPFSDKClient goelf.BpfSDKClient, policyEndpointeBPFContext *sync.Map, globalMaps *sync.Map, updateIngressProbe,
379-
updateEgressProbe, updateEventsProbe bool) (bool, bool, int, map[string]string, map[string]string, error) {
380-
isConntrackMapPresent, isPolicyEventsMapPresent := false, false
421+
updateEgressProbe, updateEventsProbe bool) (bool, bool, bool, int, map[string]string, map[string]string, error) {
422+
isConntrackMapPresent, isPolicyEventsMapPresent, isPolicyEventsScopeMapPresent := false, false, false
381423
eventsMapFD := 0
382424
var interfaceNametoIngressPinPath = make(map[string]string)
383425
var interfaceNametoEgressPinPath = make(map[string]string)
@@ -389,7 +431,7 @@ func (l *bpfClient) recoverBPFState(bpfTCClient tc.BpfTc, eBPFSDKClient goelf.Bp
389431
if err != nil {
390432
log().Errorf("failed to recover global maps %v", err)
391433
sdkAPIErr.WithLabelValues("RecoverGlobalMaps").Inc()
392-
return isConntrackMapPresent, isPolicyEventsMapPresent, eventsMapFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, nil
434+
return isConntrackMapPresent, isPolicyEventsMapPresent, isPolicyEventsScopeMapPresent, eventsMapFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, nil
393435
}
394436
log().Infof("Total no of global maps recovered count: %d", len(recoveredGlobalMaps))
395437
for globalMapName, globalMap := range recoveredGlobalMaps {
@@ -404,6 +446,11 @@ func (l *bpfClient) recoverBPFState(bpfTCClient tc.BpfTc, eBPFSDKClient goelf.Bp
404446
eventsMapFD = int(globalMap.MapFD)
405447
log().Infof("Policy event Map is already present on the node Recovered FD: %d", eventsMapFD)
406448
}
449+
if globalMapName == POLICY_EVENTS_SCOPE_MAP_PIN_PATH {
450+
log().Info("Policy event scope Map is already present on the node")
451+
isPolicyEventsScopeMapPresent = true
452+
globalMaps.Store(globalMapName, globalMap)
453+
}
407454
}
408455
}
409456

@@ -472,7 +519,7 @@ func (l *bpfClient) recoverBPFState(bpfTCClient tc.BpfTc, eBPFSDKClient goelf.Bp
472519
if err != nil {
473520
log().Errorf("GetAllBpfProgramsAndMaps failed %v", err)
474521
sdkAPIErr.WithLabelValues("GetAllBpfProgramsAndMaps").Inc()
475-
return isConntrackMapPresent, isPolicyEventsMapPresent, eventsMapFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, err
522+
return isConntrackMapPresent, isPolicyEventsMapPresent, isPolicyEventsScopeMapPresent, eventsMapFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, err
476523
}
477524
log().Infof("GetAllBpfProgramsAndMaps returned %d", len(bpfState))
478525
progIdToPinPath := make(map[int]string)
@@ -510,7 +557,7 @@ func (l *bpfClient) recoverBPFState(bpfTCClient tc.BpfTc, eBPFSDKClient goelf.Bp
510557
log().Info("Collected all data for reattaching probes")
511558
}
512559

513-
return isConntrackMapPresent, isPolicyEventsMapPresent, eventsMapFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, nil
560+
return isConntrackMapPresent, isPolicyEventsMapPresent, isPolicyEventsScopeMapPresent, eventsMapFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, nil
514561
}
515562

516563
func (l *bpfClient) ReAttachEbpfProbes() error {

‎pkg/ebpf/c/tc.v4egress.bpf.c‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,10 @@ struct pod_state {
8383
__u8 state; // 0 => POLICIES_APPLIED, 1 => DEFAULT_ALLOW, 2 => DEFAULT_DENY
8484
};
8585

86+
struct policy_scope {
87+
__u8 scope;
88+
};
89+
8690
struct bpf_map_def_pvt SEC("maps") egress_pod_state_map = {
8791
.type = BPF_MAP_TYPE_HASH,
8892
.key_size = sizeof(__u32), // default key = 0. We are storing a single state per pod identifier
@@ -94,20 +98,29 @@ struct bpf_map_def_pvt SEC("maps") egress_pod_state_map = {
9498

9599
struct bpf_map_def_pvt aws_conntrack_map;
96100
struct bpf_map_def_pvt policy_events;
101+
struct bpf_map_def_pvt policy_events_scope;
102+
103+
static void publishPolicyEvent(struct data_t evt) {
104+
__u32 plsc_key = 0;
105+
struct policy_scope *plsc = bpf_map_lookup_elem(&policy_events_scope, &plsc_key);
106+
if (plsc == NULL || plsc->scope >= evt.verdict) {
107+
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
108+
}
109+
}
97110

98111
static __always_inline int evaluateByLookUp(struct keystruct trie_key, struct conntrack_key flow_key, struct pod_state *pst, struct data_t evt) {
99112
//Check if it's in the allowed list
100113
struct lpm_trie_val *trie_val = bpf_map_lookup_elem(&egress_map, &trie_key);
101114
if (trie_val == NULL) {
102115
evt.verdict = 0;
103-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
116+
publishPolicyEvent(evt);
104117
return BPF_DROP;
105118
}
106119

107120
for (int i = 0; i < MAX_PORT_PROTOCOL; i++, trie_val++){
108121
if (trie_val->protocol == RESERVED_IP_PROTOCOL) {
109122
evt.verdict = 0;
110-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
123+
publishPolicyEvent(evt);
111124
return BPF_DROP;
112125
}
113126

@@ -137,12 +150,12 @@ static __always_inline int evaluateByLookUp(struct keystruct trie_key, struct co
137150
}
138151
bpf_map_update_elem(&aws_conntrack_map, &flow_key, &new_flow_val, 0); // 0 - BPF_ANY
139152
evt.verdict = 1;
140-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
153+
publishPolicyEvent(evt);
141154
return BPF_OK;
142155
}
143156
}
144157
evt.verdict = 0;
145-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
158+
publishPolicyEvent(evt);
146159
return BPF_DROP;
147160
}
148161

@@ -164,7 +177,6 @@ int handle_egress(struct __sk_buff *skb)
164177
__builtin_memset(&src_ip, 0, sizeof(src_ip));
165178
__builtin_memset(&reverse_flow_key, 0, sizeof(reverse_flow_key));
166179

167-
168180
struct ethhdr *ether = data;
169181
if (data + sizeof(*ether) > data_end) {
170182
return BPF_OK;
@@ -240,13 +252,13 @@ int handle_egress(struct __sk_buff *skb)
240252
// There should always be an entry in pod_state_map. pst returned in above line should never be null.
241253
if (pst == NULL) {
242254
evt.verdict = 0;
243-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
255+
publishPolicyEvent(evt);
244256
return BPF_DROP;
245257
}
246258

247259
if (pst->state == DEFAULT_DENY) {
248260
evt.verdict = 0;
249-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
261+
publishPolicyEvent(evt);
250262
return BPF_DROP;
251263
}
252264

@@ -288,6 +300,7 @@ int handle_egress(struct __sk_buff *skb)
288300
reverse_flow_val = bpf_map_lookup_elem(&aws_conntrack_map, &reverse_flow_key);
289301

290302
if (reverse_flow_val != NULL) {
303+
publishPolicyEvent(evt);
291304
return BPF_OK;
292305
}
293306

@@ -296,7 +309,7 @@ int handle_egress(struct __sk_buff *skb)
296309
new_flow_val.val = CT_VAL_DEFAULT_ALLOW;
297310
bpf_map_update_elem(&aws_conntrack_map, &flow_key, &new_flow_val, 0); // 0 - BPF_ANY
298311
evt.verdict = 1;
299-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
312+
publishPolicyEvent(evt);
300313
return BPF_OK;
301314
}
302315

‎pkg/ebpf/c/tc.v4ingress.bpf.c‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,10 @@ struct pod_state {
8383
__u8 state; // 0 => POLICIES_APPLIED, 1 => DEFAULT_ALLOW, 2 => DEFAULT_DENY
8484
};
8585

86+
struct policy_scope {
87+
__u8 scope;
88+
};
89+
8690
struct bpf_map_def_pvt SEC("maps") ingress_pod_state_map = {
8791
.type = BPF_MAP_TYPE_HASH,
8892
.key_size = sizeof(__u32), // default key = 0. We are storing a single state per pod identifier
@@ -94,20 +98,29 @@ struct bpf_map_def_pvt SEC("maps") ingress_pod_state_map = {
9498

9599
struct bpf_map_def_pvt aws_conntrack_map;
96100
struct bpf_map_def_pvt policy_events;
101+
struct bpf_map_def_pvt policy_events_scope;
102+
103+
static void publishPolicyEvent(struct data_t evt) {
104+
__u32 plsc_key = 0;
105+
struct policy_scope *plsc = bpf_map_lookup_elem(&policy_events_scope, &plsc_key);
106+
if (plsc == NULL || plsc->scope >= evt.verdict) {
107+
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
108+
}
109+
}
97110

98111
static __always_inline int evaluateByLookUp(struct keystruct trie_key, struct conntrack_key flow_key, struct pod_state *pst, struct data_t evt) {
99112
//Check if it's in the allowed list
100113
struct lpm_trie_val *trie_val = bpf_map_lookup_elem(&ingress_map, &trie_key);
101114
if (trie_val == NULL) {
102-
evt.verdict = 0;
103-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
115+
evt.verdict = 0;
116+
publishPolicyEvent(evt);
104117
return BPF_DROP;
105118
}
106119

107120
for (int i = 0; i < MAX_PORT_PROTOCOL; i++, trie_val++){
108121
if (trie_val->protocol == RESERVED_IP_PROTOCOL) {
109-
evt.verdict = 0;
110-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
122+
evt.verdict = 0;
123+
publishPolicyEvent(evt);
111124
return BPF_DROP;
112125
}
113126

@@ -136,12 +149,12 @@ static __always_inline int evaluateByLookUp(struct keystruct trie_key, struct co
136149
}
137150
bpf_map_update_elem(&aws_conntrack_map, &flow_key, &new_flow_val, 0); // 0 - BPF_ANY
138151
evt.verdict = 1;
139-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
152+
publishPolicyEvent(evt);
140153
return BPF_OK;
141154
}
142155
}
143-
evt.verdict = 0;
144-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
156+
evt.verdict = 0;
157+
publishPolicyEvent(evt);
145158
return BPF_DROP;
146159
}
147160

@@ -239,13 +252,13 @@ int handle_ingress(struct __sk_buff *skb)
239252
// There should always be an entry in pod_state_map. pst returned in above line should never be null.
240253
if (pst == NULL) {
241254
evt.verdict = 0;
242-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
255+
publishPolicyEvent(evt);
243256
return BPF_DROP;
244257
}
245258

246259
if (pst->state == DEFAULT_DENY) {
247260
evt.verdict = 0;
248-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
261+
publishPolicyEvent(evt);
249262
return BPF_DROP;
250263
}
251264

@@ -295,7 +308,7 @@ int handle_ingress(struct __sk_buff *skb)
295308
new_flow_val.val = CT_VAL_DEFAULT_ALLOW;
296309
bpf_map_update_elem(&aws_conntrack_map, &flow_key, &new_flow_val, 0); // 0 - BPF_ANY
297310
evt.verdict = 1;
298-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
311+
publishPolicyEvent(evt);
299312
return BPF_OK;
300313
}
301314

0 commit comments

Comments
 (0)