Skip to content

Commit 3dfcb11

Browse files
m00leculem00lecule
committed
feat: introduce policy event logs scope
1 parent 04cedce commit 3dfcb11

File tree

10 files changed

+170
-50
lines changed

10 files changed

+170
-50
lines changed

‎main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ func main() {
141141

142142
npMode, isMultiNICEnabled := lo.Must2(getNetworkPolicyConfigsFromIpamd(log))
143143

144-
ebpfClient := lo.Must1(ebpf.NewBpfClient(nodeIP, ctrlConfig.EnablePolicyEventLogs, ctrlConfig.EnableCloudWatchLogs,
144+
ebpfClient := lo.Must1(ebpf.NewBpfClient(nodeIP, ctrlConfig.EnablePolicyEventLogs, ctrlConfig.PolicyEventLogsScope, ctrlConfig.EnableCloudWatchLogs,
145145
ctrlConfig.EnableIPv6, ctrlConfig.ConntrackCacheCleanupPeriod, ctrlConfig.ConntrackCacheTableSize, npMode, isMultiNICEnabled))
146146
ebpfClient.ReAttachEbpfProbes()
147147

‎pkg/config/controller_config.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ const (
2121
defaultMaxConcurrentReconciles = 3
2222
defaultConntrackCacheCleanupPeriod = 300
2323
defaultConntrackCacheTableSize = 512 * 1024
24+
defaultPolicyEventLogsScope = "ACCEPT"
2425
flagEnablePolicyEventLogs = "enable-policy-event-logs"
26+
flagPolicyEventLogsScope = "policy-event-logs-scope"
2527
flagEnableCloudWatchLogs = "enable-cloudwatch-logs"
2628
flagEnableIPv6 = "enable-ipv6"
2729
flagEnableNetworkPolicy = "enable-network-policy"
@@ -58,6 +60,8 @@ type ControllerConfig struct {
5860
RuntimeConfig RuntimeConfig
5961
// Configuration for enabling profiling
6062
EnableProfiling bool
63+
// Policy event logs scope
64+
PolicyEventLogsScope string
6165
}
6266

6367
func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) {
@@ -80,7 +84,8 @@ func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) {
8084
"Cleanup interval for network policy agent conntrack cache")
8185
fs.IntVar(&cfg.ConntrackCacheTableSize, flagConntrackCacheTableSize, defaultConntrackCacheTableSize, ""+
8286
"Table size for network policy agent conntrack cache")
83-
87+
fs.StringVar(&cfg.PolicyEventLogsScope, flagPolicyEventLogsScope, defaultPolicyEventLogsScope, ""+
88+
"Set the policy event logs scope, if set to ACCEPT both ACCEPT and DENY events are generated, if set to DENY only DENY events are generated - ACCEPT, DENY")
8489
cfg.RuntimeConfig.BindFlags(fs)
8590
}
8691

‎pkg/ebpf/bpf_client.go‎

Lines changed: 55 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,15 @@ var (
4545
IPv6_HOST_MASK = "/128"
4646
CONNTRACK_MAP_PIN_PATH = "/sys/fs/bpf/globals/aws/maps/global_aws_conntrack_map"
4747
POLICY_EVENTS_MAP_PIN_PATH = "/sys/fs/bpf/globals/aws/maps/global_policy_events"
48+
POLICY_EVENTS_SCOPE_MAP_PIN_PATH = "/sys/fs/bpf/globals/aws/maps/global_policy_events_scope"
4849
CATCH_ALL_PROTOCOL corev1.Protocol = "ANY_IP_PROTOCOL"
4950
POD_VETH_PREFIX = "eni"
5051
POLICIES_APPLIED = 0
5152
DEFAULT_ALLOW = 1
5253
DEFAULT_DENY = 2
5354
POD_STATE_MAP_KEY = 0
5455
CLUSTER_POLICY_POD_STATE_MAP_KEY = 1
56+
POLICY_EVENTS_SCOPE_MAP_KEY = 0
5557
BRANCH_ENI_VETH_PREFIX = "vlan"
5658
INTERFACE_COUNT_UNKNOWN = -1 // Used when caller doesn't know interface count
5759
INTERFACE_COUNT_DEFAULT = 1 // Default single interface
@@ -85,6 +87,10 @@ type pod_state struct {
8587
state uint8
8688
}
8789

90+
type policy_scope struct {
91+
scope uint8
92+
}
93+
8894
func msSince(start time.Time) float64 {
8995
return float64(time.Since(start) / time.Millisecond)
9096
}
@@ -115,9 +121,10 @@ type BPFContext struct {
115121
conntrackMapInfo goebpfmaps.BpfMap
116122
}
117123

118-
func NewBpfClient(nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs bool,
124+
func NewBpfClient(nodeIP string, enablePolicyEventLogs bool, policyEventsLogsScope string, enableCloudWatchLogs bool,
119125
enableIPv6 bool, conntrackTTL int, conntrackTableSize int, networkPolicyMode string, isMultiNICEnabled bool) (*bpfClient, error) {
120126
var conntrackMap goebpfmaps.BpfMap
127+
var policyEventsScopeMap goebpfmaps.BpfMap
121128

122129
ebpfClient := &bpfClient{
123130
// Maps PolicyEndpoint resource to it's eBPF context
@@ -146,7 +153,7 @@ func NewBpfClient(nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs boo
146153
ebpfClient.hostMask = ingressBinary, egressBinary, hostMask
147154

148155
bpfBinaries := []string{eventsBinary, ingressBinary, egressBinary, cliBinary}
149-
isConntrackMapPresent, isPolicyEventsMapPresent := false, false
156+
isConntrackMapPresent, isPolicyEventsMapPresent, isPolicyEventsScopeMapPresent := false, false, false
150157
var err error
151158

152159
ebpfClient.bpfSDKClient = goelf.New()
@@ -181,7 +188,7 @@ func NewBpfClient(nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs boo
181188
var interfaceNametoIngressPinPath map[string]string
182189
var interfaceNametoEgressPinPath map[string]string
183190
eventBufferFD := 0
184-
isConntrackMapPresent, isPolicyEventsMapPresent, eventBufferFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, err = ebpfClient.recoverBPFState(ebpfClient.bpfTCClient, ebpfClient.bpfSDKClient, ebpfClient.policyEndpointeBPFContext,
191+
isConntrackMapPresent, isPolicyEventsMapPresent, isPolicyEventsScopeMapPresent, eventBufferFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, err = ebpfClient.recoverBPFState(ebpfClient.bpfTCClient, ebpfClient.bpfSDKClient, ebpfClient.policyEndpointeBPFContext,
185192
ebpfClient.globalMaps, ingressUpdateRequired, egressUpdateRequired, eventsUpdateRequired)
186193
if err != nil {
187194
//Log the error and move on
@@ -196,7 +203,7 @@ func NewBpfClient(nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs boo
196203
// - Current events binary packaged with network policy agent is different than the one installed
197204
// during the previous installation (or)
198205
// - Either Conntrack Map (or) Events Map is currently missing on the node
199-
if eventsUpdateRequired || (!isConntrackMapPresent || !isPolicyEventsMapPresent) {
206+
if eventsUpdateRequired || (!isConntrackMapPresent || !isPolicyEventsMapPresent || !isPolicyEventsScopeMapPresent) {
200207
log().Info("Install the default global maps")
201208
eventsProbe := EVENTS_BINARY
202209
if enableIPv6 {
@@ -226,6 +233,10 @@ func NewBpfClient(nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs boo
226233
if mapName == AWS_EVENTS_MAP {
227234
eventBufferFD = int(mapInfo.MapFD)
228235
}
236+
if mapName == AWS_EVENTS_SCOPE_MAP {
237+
policyEventsScopeMap = mapInfo
238+
isPolicyEventsScopeMapPresent = true
239+
}
229240
}
230241
}
231242

@@ -244,6 +255,36 @@ func NewBpfClient(nodeIP string, enablePolicyEventLogs, enableCloudWatchLogs boo
244255
ebpfClient.conntrackClient = conntrack.NewConntrackClient(conntrackMap, enableIPv6)
245256
log().Info("Initialized Conntrack client")
246257

258+
//if present update the PolicyEventsScope Map
259+
if isPolicyEventsScopeMapPresent {
260+
recoveredPolicyEventsScopeMap, ok := ebpfClient.globalMaps.Load(POLICY_EVENTS_SCOPE_MAP_PIN_PATH)
261+
if ok {
262+
policyEventsScopeMap = recoveredPolicyEventsScopeMap.(goebpfmaps.BpfMap)
263+
log().Info("Derived existing policyEventsScopeMap identifier")
264+
} else {
265+
log().Errorf("Unable to get policyEventsScopeMap post recovery..error: %v", err)
266+
sdkAPIErr.WithLabelValues("RecoveryFailed").Inc()
267+
return nil, err
268+
}
269+
270+
key := uint32(POLICY_EVENTS_SCOPE_MAP_KEY)
271+
scope := uint8(utils.ACCEPT.Index())
272+
273+
if policyEventsLogsScope == "DENY" {
274+
scope = uint8(utils.DENY.Index())
275+
}
276+
277+
value := policy_scope{scope: scope}
278+
log().Infof("Will update Policy Events Scope Map: key=%d value=%v", key, value)
279+
err := policyEventsScopeMap.CreateUpdateMapEntry(uintptr(unsafe.Pointer(&key)), uintptr(unsafe.Pointer(&value)), 0)
280+
281+
if err != nil {
282+
log().Errorf("Policy Events Scope Map update failed: %v", err)
283+
sdkAPIErr.WithLabelValues("updateEbpfMap-policy-events-scope").Inc()
284+
}
285+
log().Infof("Updated Policy Events Scope Map: key=%d value=%v", key, value)
286+
}
287+
247288
if enablePolicyEventLogs {
248289
err = events.ConfigurePolicyEventsLogging(enableCloudWatchLogs, eventBufferFD, enableIPv6)
249290
if err != nil {
@@ -379,8 +420,8 @@ func checkAndUpdateBPFBinaries(bpfTCClient tc.BpfTc, bpfBinaries []string, hostB
379420
}
380421

381422
func (l *bpfClient) recoverBPFState(bpfTCClient tc.BpfTc, eBPFSDKClient goelf.BpfSDKClient, policyEndpointeBPFContext *sync.Map, globalMaps *sync.Map, updateIngressProbe,
382-
updateEgressProbe, updateEventsProbe bool) (bool, bool, int, map[string]string, map[string]string, error) {
383-
isConntrackMapPresent, isPolicyEventsMapPresent := false, false
423+
updateEgressProbe, updateEventsProbe bool) (bool, bool, bool, int, map[string]string, map[string]string, error) {
424+
isConntrackMapPresent, isPolicyEventsMapPresent, isPolicyEventsScopeMapPresent := false, false, false
384425
eventsMapFD := 0
385426
var interfaceNametoIngressPinPath = make(map[string]string)
386427
var interfaceNametoEgressPinPath = make(map[string]string)
@@ -392,7 +433,7 @@ func (l *bpfClient) recoverBPFState(bpfTCClient tc.BpfTc, eBPFSDKClient goelf.Bp
392433
if err != nil {
393434
log().Errorf("failed to recover global maps %v", err)
394435
sdkAPIErr.WithLabelValues("RecoverGlobalMaps").Inc()
395-
return isConntrackMapPresent, isPolicyEventsMapPresent, eventsMapFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, nil
436+
return isConntrackMapPresent, isPolicyEventsMapPresent, isPolicyEventsScopeMapPresent, eventsMapFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, nil
396437
}
397438
log().Infof("Total no of global maps recovered count: %d", len(recoveredGlobalMaps))
398439
for globalMapName, globalMap := range recoveredGlobalMaps {
@@ -407,6 +448,11 @@ func (l *bpfClient) recoverBPFState(bpfTCClient tc.BpfTc, eBPFSDKClient goelf.Bp
407448
eventsMapFD = int(globalMap.MapFD)
408449
log().Infof("Policy event Map is already present on the node Recovered FD: %d", eventsMapFD)
409450
}
451+
if globalMapName == POLICY_EVENTS_SCOPE_MAP_PIN_PATH {
452+
log().Info("Policy event scope Map is already present on the node")
453+
isPolicyEventsScopeMapPresent = true
454+
globalMaps.Store(globalMapName, globalMap)
455+
}
410456
}
411457
}
412458

@@ -505,7 +551,7 @@ func (l *bpfClient) recoverBPFState(bpfTCClient tc.BpfTc, eBPFSDKClient goelf.Bp
505551
if err != nil {
506552
log().Errorf("GetAllBpfProgramsAndMaps failed %v", err)
507553
sdkAPIErr.WithLabelValues("GetAllBpfProgramsAndMaps").Inc()
508-
return isConntrackMapPresent, isPolicyEventsMapPresent, eventsMapFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, err
554+
return isConntrackMapPresent, isPolicyEventsMapPresent, isPolicyEventsScopeMapPresent, eventsMapFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, err
509555
}
510556
log().Infof("GetAllBpfProgramsAndMaps returned %d", len(bpfState))
511557
progIdToPinPath := make(map[int]string)
@@ -543,7 +589,7 @@ func (l *bpfClient) recoverBPFState(bpfTCClient tc.BpfTc, eBPFSDKClient goelf.Bp
543589
log().Info("Collected all data for reattaching probes")
544590
}
545591

546-
return isConntrackMapPresent, isPolicyEventsMapPresent, eventsMapFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, nil
592+
return isConntrackMapPresent, isPolicyEventsMapPresent, isPolicyEventsScopeMapPresent, eventsMapFD, interfaceNametoIngressPinPath, interfaceNametoEgressPinPath, nil
547593
}
548594

549595
func (l *bpfClient) ReAttachEbpfProbes() error {

‎pkg/ebpf/c/tc.v4egress.bpf.c‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,10 @@ struct pod_state {
125125
__u8 state; // 0 => POLICIES_APPLIED, 1 => DEFAULT_ALLOW, 2 => DEFAULT_DENY
126126
};
127127

128+
struct policy_scope {
129+
__u8 scope;
130+
};
131+
128132
struct bpf_map_def_pvt SEC("maps") egress_pod_state_map = {
129133
.type = BPF_MAP_TYPE_HASH,
130134
.key_size = sizeof(__u32), // network policy key 0, cluster policy key 1
@@ -136,6 +140,15 @@ struct bpf_map_def_pvt SEC("maps") egress_pod_state_map = {
136140

137141
struct bpf_map_def_pvt aws_conntrack_map;
138142
struct bpf_map_def_pvt policy_events;
143+
struct bpf_map_def_pvt policy_events_scope;
144+
145+
static void publishPolicyEvent(struct data_t *evt) {
146+
__u32 plsc_key = 0;
147+
struct policy_scope *plsc = bpf_map_lookup_elem(&policy_events_scope, &plsc_key);
148+
if (plsc == NULL || plsc->scope >= evt->verdict) {
149+
bpf_ringbuf_output(&policy_events, evt, sizeof(*evt), 0);
150+
}
151+
}
139152

140153
static __always_inline int evaluateClusterPolicyByLookUp(struct keystruct trie_key, struct conntrack_key flow_key, __u32 *admin_tier_priority, __u8 *baseline_tier_action, __u32 *baseline_tier_priority) {
141154

@@ -254,15 +267,15 @@ static __always_inline int evaluateFlow(struct keystruct trie_key, struct conntr
254267
case ACTION_DENY: {
255268
evt->verdict = 0;
256269
evt->tier = ADMIN_TIER;
257-
bpf_ringbuf_output(&policy_events, evt, sizeof(*evt), 0);
270+
publishPolicyEvent(evt);
258271
return BPF_DROP;
259272
}
260273
case ACTION_ALLOW: {
261274
flow_val.val = pod_state_val;
262275
bpf_map_update_elem(&aws_conntrack_map, &flow_key, &flow_val, 0);
263276
evt->verdict = 1;
264277
evt->tier = ADMIN_TIER;
265-
bpf_ringbuf_output(&policy_events, evt, sizeof(*evt), 0);
278+
publishPolicyEvent(evt);
266279
return BPF_OK;
267280
}
268281
default:
@@ -278,13 +291,13 @@ static __always_inline int evaluateFlow(struct keystruct trie_key, struct conntr
278291
bpf_map_update_elem(&aws_conntrack_map, &flow_key, &flow_val, 0); // 0 - BPF_ANY
279292
evt->verdict = 1;
280293
evt->tier = NETWORK_POLICY_TIER;
281-
bpf_ringbuf_output(&policy_events, evt, sizeof(*evt), 0);
294+
publishPolicyEvent(evt);
282295
return BPF_OK;
283296
}
284297
case ACTION_DENY:{
285298
evt->verdict = 0;
286299
evt->tier = NETWORK_POLICY_TIER;
287-
bpf_ringbuf_output(&policy_events, evt, sizeof(*evt), 0);
300+
publishPolicyEvent(evt);
288301
return BPF_DROP;
289302
}
290303
case ACTION_PASS:
@@ -296,15 +309,15 @@ static __always_inline int evaluateFlow(struct keystruct trie_key, struct conntr
296309
case ACTION_DENY: {
297310
evt->verdict = 0;
298311
evt->tier = BASELINE_TIER;
299-
bpf_ringbuf_output(&policy_events, evt, sizeof(*evt), 0);
312+
publishPolicyEvent(evt);
300313
return BPF_DROP;
301314
}
302315
case ACTION_ALLOW: {
303316
flow_val.val = pod_state_val;
304317
bpf_map_update_elem(&aws_conntrack_map, &flow_key, &flow_val, 0);
305318
evt->verdict = 1;
306319
evt->tier = BASELINE_TIER;
307-
bpf_ringbuf_output(&policy_events, evt, sizeof(*evt), 0);
320+
publishPolicyEvent(evt);
308321
return BPF_OK;
309322
}
310323
case ACTION_PASS: {
@@ -314,13 +327,13 @@ static __always_inline int evaluateFlow(struct keystruct trie_key, struct conntr
314327
bpf_map_update_elem(&aws_conntrack_map, &flow_key, &flow_val, 0);
315328
evt->verdict = 1;
316329
evt->tier = DEFAULT_TIER;
317-
bpf_ringbuf_output(&policy_events, evt, sizeof(*evt), 0);
330+
publishPolicyEvent(evt);
318331
return BPF_OK;
319332
}
320333
case DEFAULT_DENY: {
321334
evt->verdict = 0;
322335
evt->tier = DEFAULT_TIER;
323-
bpf_ringbuf_output(&policy_events, evt, sizeof(*evt), 0);
336+
publishPolicyEvent(evt);
324337
return BPF_DROP;
325338
}
326339
}
@@ -349,7 +362,6 @@ int handle_egress(struct __sk_buff *skb)
349362
__builtin_memset(&src_ip, 0, sizeof(src_ip));
350363
__builtin_memset(&reverse_flow_key, 0, sizeof(reverse_flow_key));
351364

352-
353365
struct ethhdr *ether = data;
354366
if (data + sizeof(*ether) > data_end) {
355367
return BPF_OK;
@@ -431,7 +443,7 @@ int handle_egress(struct __sk_buff *skb)
431443
if ((pst == NULL) || (clusterpolicy_pst == NULL)) {
432444
evt.verdict = 0;
433445
evt.tier = ERROR_TIER;
434-
bpf_ringbuf_output(&policy_events, &evt, sizeof(evt), 0);
446+
publishPolicyEvent(&evt);
435447
return BPF_DROP;
436448
}
437449

0 commit comments

Comments
 (0)