gcp: add support for parsing sensitive action notifications event in audit dataset.

This enables ingestion and processing of sensitive audit events[1], improving
visibility and tracking of critical actions in the system.

Test samples have been derived from the issue[2].

[1] https://cloud.google.com/advisory-notifications/docs/sensitive-actions-overview
[2] #14225

Author: mohitjha-elastic

packages/gcp/_dev/deploy/docker/sample_logs/audit.log

@@ -5,3 +5,4 @@
 {"insertId":"87efd529-6349-45d2-b905-fc607e6c5d3b","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cert-manager-webhook:auth-delegator\" of ClusterRole \"system:auth-delegator\" to ServiceAccount \"cert-manager-webhook/cert-manager\""},"logName":"projects/foo/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"5555555-6349-45d2-b905-fc607e6c5d3b","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:serviceaccount:cert-manager:cert-manager-webhook"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","resource":"authorization.k8s.io/v1beta1/subjectaccessreviews"}],"methodName":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","request":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":false}},"requestMetadata":{"callerIp":"10.11.12.13","callerSuppliedUserAgent":"webhook/v0.0.0 (linux/amd64) kubernetes/$Format"},"resourceName":"authorization.k8s.io/v1beta1/subjectaccessreviews","response":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":true,"reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""}},"serviceName":"k8s.io","status":{"code":0}},"receiveTimestamp":"2020-08-05T21:07:32.157698684Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2020-08-05T21:07:30.974750Z"}
 {"insertId":"v2spcwdzmc2","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"first":true,"id":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"[email protected]"},"authorizationInfo":[{"granted":true,"permission":"compute.images.create","resourceAttributes":{"name":"projects/foo/global/images/windows-server-2016-v20200805","service":"compute","type":"compute.images"}}],"methodName":"v1.compute.images.insert","request":{"@type":"type.googleapis.com/compute.images.insert","family":"windows-server-2016","guestOsFeatures":[{"type":"VIRTIO_SCSI_MULTIQUEUE"},{"type":"WINDOWS"}],"name":"windows-server-2016-v20200805","rawDisk":{"source":"https://storage.googleapis.com/storage/v1/b/foo/o/windows-server-2016-v20200805.tar.gz"},"sourceType":"RAW"},"requestMetadata":{"callerIp":"67.43.156.13","callerSuppliedUserAgent":"google-cloud-sdk gcloud/290.0.1 command/gcloud.compute.images.create invocation-id/032752ad0fa44b4ea951951d2deef6a3 environment/None environment-version/None interactive/True from-script/False python/2.7.17 term/xterm-256color (Macintosh; Intel Mac OS X 19.6.0),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2020-08-05T21:59:27.515Z"}},"resourceLocation":{"currentLocations":["eu"]},"resourceName":"projects/foo/global/images/windows-server-2016-v20200805","response":{"@type":"type.googleapis.com/operation","id":"44919313","insertTime":"2020-08-05T14:59:27.259-07:00","name":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","operationType":"insert","progress":"0","selfLink":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/operation-1596664766354-5ac287c395484-fa3923bd-543e018e","selfLinkWithId":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/4491931805423146320","startTime":"2020-08-05T14:59:27.274-07:00","status":"RUNNING","targetId":"12345","targetLink":"https://www.googleapis.com/compute/v1/projects/foo/global/images/windows-server-2016-v20200805","user":"[email protected]"},"serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T21:59:27.822546978Z","resource":{"labels":{"image_id":"771879043","project_id":"foo"},"type":"gce_image"},"severity":"NOTICE","timestamp":"2020-08-05T21:59:26.456Z"}
 {"insertId":"-c7ctxmd2zab","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"id":"operation-1596646123456-5ac2438b775f6-f8ca1382-e70b6831","last":true,"producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"[email protected]"},"methodName":"beta.compute.instances.stop","request":{"@type":"type.googleapis.com/compute.instances.stop"},"requestMetadata":{"callerIp":"67.43.156.13","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:79.0) Gecko/20100101 Firefox/79.0,gzip(gfe),gzip(gfe)"},"resourceName":"projects/foo/zones/us-central1-a/instances/win10-test","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T16:56:41.315135528Z","resource":{"labels":{"instance_id":"590261181","project_id":"foo","zone":"us-central1-a"},"type":"gce_instance"},"severity":"NOTICE","timestamp":"2020-08-05T16:56:40.428Z"}
+{"insertId":"veqbqlc5a2","jsonPayload":{"access":{"callerIp":"89.160.20.156","callerIpGeo":{"regionCode":"DE"},"methodName":"v1.compute.projects.setCommonInstanceMetadata","principalEmail":"[email protected]","principalSubject":"user:[email protected]","serviceName":"compute.googleapis.com","userAgent":"google-cloud-sdk gcloud/525.0.0 command/gcloud.compute.ssh invocation-id/881a0585a52e4ff68bbaf3efb4f2da2c environment/None environment-version/None client-os/MACOSX client-os-ver/24.5.0 client-pltf-arch/arm interactive/True from-script/False python/3.13.4 term/xterm-256color (Macintosh; Intel Mac OS X 24.5.0),gzip(gfe)"},"actionTime":"2025-06-13T13:42:47.922290Z","actionType":"add_ssh_key","affectedResources":["//compute.googleapis.com/projects/10594"],"learnMoreUri":"https://cloud.google.com/security-command-center/docs/concepts-sensitive-actions-overview","sourceLogIds":[{"insertId":"3czf3od3","logTime":"2025-06-13T13:42:46.293601Z","queryUri":"https://console.cloud.google.com/logs/query;query=timestamp%3D%222025-06-13T13:42:46.293601Z%22%0AinsertId%3D%223czf3od3k16%22?project=elastic-sa","resourceContainer":"projects/elastic-sa"}]},"logName":"projects/elastic-sa/logs/sensitiveaction.googleapis.com%2Faction","receiveTimestamp":"2025-06-13T13:42:49.295214268Z","resource":{"labels":{"location":"global","resource_container":"//compute.googleapis.com/projects/10594"},"type":"sensitiveaction.googleapis.com/Location"},"severity":"NOTICE","timestamp":"2025-06-13T13:42:47.92229Z"}

packages/gcp/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 2.44.0
+  changes:
+    - description: Add support for parsing sensitive action notifications event in the audit dataset.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15619
 - version: "2.43.0"
   changes:
     - description: Add tags and processors to GCP Billing, Cloudrun, CloudSQL, Dataproc, GKE, Loadbalancing, Pubsub and Redis.

packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log

@@ -21,3 +21,4 @@
 {"insertId":"15ciwwfd47gf","logName":"projects/elastic/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"[email protected]","principalSubject":"serviceAccount:[email protected]"},"authorizationInfo":[{"granted": true,"permission":"container.clusters.get","resourceAttributes":{}}],"methodName":"google.container.v1.ClusterManager.GetCluster","policyViolationInfo":{"orgPolicyViolationInfo":{"payload":{"key1":"value1","key2":"value2"},"resourceType":"compute.googleapis.com/Instance","resourceTags":{"instance_id":"INSTANCE_ID","zone":"us-central1-a"},"violationInfo":[{"constraint":"compute.vmExternalIpAccess","errorMessage":"This policy disallows the use of external IP addresses for VM instances.","checkedValue":"Value","policyType":"CUSTOM_CONSTRAINT"}]}},"request":{"@type":"type.googleapis.com/google.container.v1alpha1.GetClusterRequest","name":"projects/elastic-product/locations/us-central1-a/clusters/demo-elastic-co","policy":"scalar-policy"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"google-api-go-client/0.5 cluster-autoscaler,gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2022-06-01T11:15:10.836131149Z"}},"resourceLocation":{"currentLocations":["us-central1-a"]},"resourceName":"projects/elastic-product/zones/us-central1-a/clusters/demo-elastic-co","serviceName":"container.googleapis.com"},"receiveTimestamp":"2022-06-01T11:15:11.07151757Z","resource":{"labels":{"cluster_name":"demo-elastic-co","location":"us-central1-a","project_id":"elastic-product"},"type":"gke_cluster"},"severity":"INFO","timestamp":"2022-06-01T11:15:10.842495409Z","logging.googleapis.com/timestamp":"2022-06-01T11:15:10.842495409Z"}
 {"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","status":{},"serviceName":"container.googleapis.com","methodName":"google.container.v1.ClusterManager.SetLabels","resourceName":"projects/elastic-siem/zones/us-central1-c/clusters/endpoint-gke-cluster","metadata":{"operationType":"UPDATE_CLUSTER"},"resourceLocation":{"currentLocations":["us-central1-c"]},"policyViolationInfo":{"orgPolicyViolationInfo":{}}},"insertId":"17ah0cpe10gvp","resource":{"type":"gke_cluster","labels":{"location":"us-central1-c","cluster_name":"endpoint-gke-cluster","project_id":"elastic-siem"}},"timestamp":"2024-08-23T02:12:01.626546355Z","severity":"NOTICE","logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Factivity","operation":{"id":"operation-1724379121483-d43ef943-bcf8-46e9-9ff2-ba71cfbc26b2","producer":"container.googleapis.com","last": true},"receiveTimestamp":"2024-08-23T02:12:02.419428097Z"}
 {"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"[email protected]"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.core.v1.pods.delete","resource":"core/v1/namespaces/default/pods/debug4"}],"methodName":"io.k8s.core.v1.pods.delete","request":{"@type":"meta.k8s.io/__internal.DeleteOptions","apiVersion":"meta.k8s.io/__internal","kind":"DeleteOptions","propagationPolicy":"Background"},"requestMetadata":{"callerIp":"67.43.156.0","callerSuppliedUserAgent":"kubectl/v1.26.3 (linux/amd64) kubernetes/9e64410"},"resourceName":"core/v1/namespaces/default/pods/debug4","response":{"@type":"core.k8s.io/v1.Pod","apiVersion":"v1","kind":"Pod","metadata":{"creationTimestamp":"2023-04-27T11:50:35Z","deletionGracePeriodSeconds":30,"deletionTimestamp":"2023-04-27T20:59:07Z","labels":{"run":"debug4"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:labels":{".":{},"f:run":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"debug4\"}":{".":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:stdin":{},"f:stdinOnce":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:tty":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}},"manager":"kubectl-run","operation":"Update","time":"2023-04-27T11:50:35Z"},{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"216.160.83.56\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"manager":"kubelet","operation":"Update","subresource":"status","time":"2023-04-27T11:50:36Z"}],"name":"debug4","namespace":"default","resourceVersion":"76620168","uid":"6661e893-99d5-4a9d-ac16-254a177e9516"},"spec":{"containers":[{"image":"ubuntu","imagePullPolicy":"Always","name":"debug4","resources":{},"stdin":true,"stdinOnce":true,"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","tty":true,"volumeMounts":[{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount","name":"kube-api-access-pls9v","readOnly":true}]}],"dnsPolicy":"ClusterFirst","enableServiceLinks":true,"hostNetwork":true,"nodeName":"gke-sa-da-gke-lls-default-pool-1e5fc85d-9rug","preemptionPolicy":"PreemptLowerPriority","priority":0,"restartPolicy":"Never","schedulerName":"default-scheduler","securityContext":{},"serviceAccount":"default","serviceAccountName":"default","terminationGracePeriodSeconds":30,"tolerations":[{"effect":"NoExecute","key":"node.kubernetes.io/not-ready","operator":"Exists","tolerationSeconds":300},{"effect":"NoExecute","key":"node.kubernetes.io/unreachable","operator":"Exists","tolerationSeconds":300}],"volumes":[{"name":"kube-api-access-pls9v","projected":{"defaultMode":420,"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"items":[{"key":"ca.crt","path":"ca.crt"}],"name":"kube-root-ca.crt"}},{"downwardAPI":{"items":[{"fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"},"path":"namespace"}]}}]}}]},"status":{"conditions":[{"lastProbeTime":null,"lastTransitionTime":"2023-04-27T11:50:35Z","status":"True","type":"Initialized"},{"lastProbeTime":null,"lastTransitionTime":"2023-04-27T11:50:36Z","status":"True","type":"Ready"},{"lastProbeTime":null,"lastTransitionTime":"2023-04-27T11:50:36Z","status":"True","type":"ContainersReady"},{"lastProbeTime":null,"lastTransitionTime":"2023-04-27T11:50:35Z","status":"True","type":"PodScheduled"}],"containerStatuses":[{"containerID":"containerd://415af393459f3a8bccabb0ddd595e3276d2b5206bc3d1f34866c5fc6e14cd35f","image":"docker.io/library/ubuntu:latest","imageID":"docker.io/library/ubuntu@sha256:67211c14fa74f070d27cc59d69a7fa9aeff8e28ea118ef3babc295a0428a6d21","lastState":{},"name":"debug4","ready":true,"restartCount":0,"started":true,"state":{"running":{"startedAt":"2023-04-27T11:50:36Z"}}}],"hostIP":"216.160.83.56","phase":"Running","podIP":"216.160.83.56","podIPs":[{"ip":"216.160.83.56"}],"qosClass":"BestEffort","startTime":"2023-04-27T11:50:35Z"}},"serviceName":"k8s.io","status":{"code":0}},"insertId":"c9f95099-6738-4781-8993-58d5fbb5c2c0","resource":{"type":"k8s_cluster","labels":{"location":"europe-west1","cluster_name":"sa-da-gke-lls","project_id":"elastic-sa"}},"timestamp":"2023-04-27T20:58:37.419997Z","labels":{"authorization.k8s.io/reason":"access granted by IAM permissions.","authorization.k8s.io/decision":"allow"},"logName":"projects/elastic-sa/logs/cloudaudit.googleapis.com%2Factivity","operation":{"id":"c9f95099-6738-4781-8993-58d5fbb5c2c0","producer":"k8s.io","first":true,"last":true},"receiveTimestamp":"2023-04-27T20:59:08.485927903Z"}
+{"insertId":"veqbqlc5a2","jsonPayload":{"access":{"callerIp":"89.160.20.156","callerIpGeo":{"regionCode":"DE"},"methodName":"v1.compute.projects.setCommonInstanceMetadata","principalEmail":"[email protected]","principalSubject":"user:[email protected]","serviceName":"compute.googleapis.com","userAgent":"google-cloud-sdk gcloud/525.0.0 command/gcloud.compute.ssh invocation-id/881a0585a52e4ff68bbaf3efb4f2da2c environment/None environment-version/None client-os/MACOSX client-os-ver/24.5.0 client-pltf-arch/arm interactive/True from-script/False python/3.13.4 term/xterm-256color (Macintosh; Intel Mac OS X 24.5.0),gzip(gfe)"},"actionTime":"2025-06-13T13:42:47.922290Z","actionType":"add_ssh_key","affectedResources":["//compute.googleapis.com/projects/10594"],"learnMoreUri":"https://cloud.google.com/security-command-center/docs/concepts-sensitive-actions-overview","sourceLogIds":[{"insertId":"3czf3od3","logTime":"2025-06-13T13:42:46.293601Z","queryUri":"https://console.cloud.google.com/logs/query;query=timestamp%3D%222025-06-13T13:42:46.293601Z%22%0AinsertId%3D%223czf3od3k16%22?project=elastic-sa","resourceContainer":"projects/elastic-sa"}]},"logName":"projects/elastic-sa/logs/sensitiveaction.googleapis.com%2Faction","receiveTimestamp":"2025-06-13T13:42:49.295214268Z","resource":{"labels":{"location":"global","resource_container":"//compute.googleapis.com/projects/10594"},"type":"sensitiveaction.googleapis.com/Location"},"severity":"NOTICE","timestamp":"2025-06-13T13:42:47.92229Z"}