Add filtering out of documents with error.message from latest indexes (#15722)

alexreal1314 · web-flow · commit 0c5f7647fdc8 · 2025-10-29T15:02:38.000+02:00
* add filtering out of documents with error.message from latest misconfiguration and vulnerability index

change is added to all supported native and 3p integrations

* bump transform versions
diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "4.3.2"
+  changes:
+    - description: Update transform to filter out document containing an error.message from AWS Config, AWS Inspector, and AWS Security Hub latest indexes.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15722
 - version: "4.3.1"
   changes:
     - description: Update the AWS CloudWatch documentation.
diff --git a/packages/aws/elasticsearch/transform/latest_cdr_misconfigurations/transform.yml b/packages/aws/elasticsearch/transform/latest_cdr_misconfigurations/transform.yml
@@ -1,6 +1,11 @@
 source:
   index:
     - "logs-aws.securityhub_findings_full_posture-*"
+  query:
+    bool:
+      must_not:
+        exists:
+          field: error.message
 dest:
   index: "security_solution-aws.misconfiguration_latest-v2"
   aliases:
@@ -27,4 +32,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.2.0
+  fleet_transform_version: 0.2.1
diff --git a/packages/aws/elasticsearch/transform/latest_cdr_misconfigurations_awsconfig/transform.yml b/packages/aws/elasticsearch/transform/latest_cdr_misconfigurations_awsconfig/transform.yml
@@ -2,6 +2,11 @@
 source:
   index:
     - "logs-aws.config-*"
+  query:
+    bool:
+      must_not:
+        exists:
+          field: error.message
 dest:
   index: "security_solution-awsconfig.misconfiguration_latest-v1"
   aliases:
@@ -30,4 +35,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during
   # package installation.
-  fleet_transform_version: 0.1.0
+  fleet_transform_version: 0.1.1
diff --git a/packages/aws/elasticsearch/transform/latest_cdr_vulnerabilities_awsinspector/transform.yml b/packages/aws/elasticsearch/transform/latest_cdr_vulnerabilities_awsinspector/transform.yml
@@ -8,6 +8,9 @@ source:
             aws.inspector.status: ACTIVE
         - match:
             aws.inspector.type: PACKAGE_VULNERABILITY
+      must_not:
+        exists:
+          field: error.message
 dest:
   index: "security_solution-awsinspector.vulnerability_latest-v1"
   aliases:
@@ -33,4 +36,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.1.0
+  fleet_transform_version: 0.1.1
diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.3.2
 name: aws
 title: AWS
-version: "4.3.1"
+version: "4.3.2"
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories:
diff --git a/packages/cloud_security_posture/changelog.yml b/packages/cloud_security_posture/changelog.yml
@@ -16,6 +16,11 @@
 # 1.4.x - 8.9.x
 # 1.3.x - 8.8.x
 # 1.2.x - 8.7.x
+- version: "3.1.1"
+  changes:
+    - description: Update transform to filter out documents containing an error message from latest vulnerability and misconfiguration indexes.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15722
 - version: "3.1.0"
   changes:
     - description: Release version 3.1.0
diff --git a/packages/cloud_security_posture/elasticsearch/transform/misconfiguration/transform.yml b/packages/cloud_security_posture/elasticsearch/transform/misconfiguration/transform.yml
@@ -1,6 +1,11 @@
 source:
   index:
     - "logs-cloud_security_posture.findings-*"
+  query:
+    bool:
+      must_not:
+        exists:
+          field: error.message
 dest:
   index: "security_solution-cloud_security_posture.misconfiguration_latest-v1"
   aliases:
@@ -27,4 +32,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.2.0
+  fleet_transform_version: 0.2.1
diff --git a/packages/cloud_security_posture/manifest.yml b/packages/cloud_security_posture/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.3.2
 name: cloud_security_posture
 title: "Security Posture Management"
-version: "3.1.0"
+version: "3.1.1"
 source:
   license: "Elastic-2.0"
 description: "Identify & remediate configuration risks in your Cloud infrastructure"
diff --git a/packages/google_scc/changelog.yml b/packages/google_scc/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.2.1"
+  changes:
+    - description: Update transform to filter out documents containing an error.message from latest vulnerability and misconfiguration indexes.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15722
 - version: "2.2.0"
   changes:
     - description: Prevent updating fleet health status to degraded.
diff --git a/packages/google_scc/elasticsearch/transform/latest_cdr_misconfigurations/transform.yml b/packages/google_scc/elasticsearch/transform/latest_cdr_misconfigurations/transform.yml
@@ -6,6 +6,9 @@ source:
       must:
         - match:
             google_scc.finding.class: MISCONFIGURATION
+      must_not:
+        exists:
+          field: error.message
 dest:
   index: "security_solution-google_scc.misconfiguration_latest-v1"
   aliases:
@@ -33,4 +36,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.1.0
+  fleet_transform_version: 0.1.1
diff --git a/packages/google_scc/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml b/packages/google_scc/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml
@@ -8,6 +8,9 @@ source:
             google_scc.finding.class: VULNERABILITY
         - match:
             google_scc.finding.state: ACTIVE
+      must_not:
+        exists:
+          field: error.message
 dest:
   index: "security_solution-google_scc.vulnerability_latest-v1"
   aliases:
@@ -36,4 +39,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.1.0
+  fleet_transform_version: 0.1.1
diff --git a/packages/google_scc/manifest.yml b/packages/google_scc/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.2.3"
 name: google_scc
 title: Google Security Command Center
-version: "2.2.0"
+version: "2.2.1"
 description: Collect logs from Google Security Command Center with Elastic Agent.
 type: integration
 categories:
diff --git a/packages/m365_defender/changelog.yml b/packages/m365_defender/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "5.0.1"
+  changes:
+    - description: Update transform to filter out documents containing an error.message from vulnerability latest index
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15722
 - version: "5.0.0"
   changes:
     - description: |
diff --git a/packages/m365_defender/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml b/packages/m365_defender/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml
@@ -9,6 +9,9 @@ source:
       filter:
         - exists:
             field: resource.id
+      must_not:
+        exists:
+          field: error.message
 dest:
   index: "security_solution-m365_defender.vulnerability_latest-v2"
   aliases:
@@ -37,4 +40,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.2.0
+  fleet_transform_version: 0.2.1
diff --git a/packages/m365_defender/manifest.yml b/packages/m365_defender/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.4.0"
 name: m365_defender
 title: Microsoft Defender XDR
-version: "5.0.0"
+version: "5.0.1"
 description: Collect logs from Microsoft Defender XDR with Elastic Agent.
 categories:
   - "security"
diff --git a/packages/microsoft_defender_cloud/changelog.yml b/packages/microsoft_defender_cloud/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.1.1"
+  changes:
+    - description: Update transform to filter out documents containing an error.message from latest vulnerability and misconfiguration indexes.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15722
 - version: "3.1.0"
   changes:
     - description: Add `vulnerability_workflow` and `misconfiguration_workflow` sub category label.
diff --git a/packages/microsoft_defender_cloud/elasticsearch/transform/misconfiguration/transform.yml b/packages/microsoft_defender_cloud/elasticsearch/transform/misconfiguration/transform.yml
@@ -6,6 +6,9 @@ source:
       must:
         - match:
             microsoft_defender_cloud.assessment.class: misconfiguration
+      must_not:
+        exists:
+          field: error.message
 dest:
   index: 'security_solution-microsoft_defender_cloud.misconfiguration_latest-v1'
   aliases:
@@ -33,4 +36,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.1.0
+  fleet_transform_version: 0.1.1
diff --git a/packages/microsoft_defender_cloud/elasticsearch/transform/vulnerability/transform.yml b/packages/microsoft_defender_cloud/elasticsearch/transform/vulnerability/transform.yml
@@ -10,6 +10,9 @@ source:
             event.outcome: failure
         - exists:
             field: package.name
+      must_not:
+        exists:
+          field: error.message
 dest:
   index: 'security_solution-microsoft_defender_cloud.vulnerability_latest-v1'
   aliases:
@@ -39,4 +42,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.1.0
+  fleet_transform_version: 0.1.1
diff --git a/packages/microsoft_defender_cloud/manifest.yml b/packages/microsoft_defender_cloud/manifest.yml
@@ -1,7 +1,7 @@
 format_version: '3.3.2'
 name: microsoft_defender_cloud
 title: Microsoft Defender for Cloud
-version: '3.1.0'
+version: '3.1.1'
 description: Collect logs from Microsoft Defender for Cloud with Elastic Agent.
 type: integration
 categories:
diff --git a/packages/microsoft_defender_endpoint/changelog.yml b/packages/microsoft_defender_endpoint/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "4.1.1"
+  changes:
+    - description: Update transform to filter out documents containing an error.message from vulnerability latest index
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15722
 - version: "4.1.0"
   changes:
     - description: Add support for OAuth2 Endpoint Params option.
diff --git a/packages/microsoft_defender_endpoint/elasticsearch/transform/latest_action/transform.yml b/packages/microsoft_defender_endpoint/elasticsearch/transform/latest_action/transform.yml
@@ -2,6 +2,11 @@
 source:
   index:
     - "logs-microsoft_defender_endpoint.machine_action-*"
+  query:
+    bool:
+      must_not:
+        - exists:
+            field: error.message
 dest:
   index: "logs-microsoft_defender_endpoint_latest.dest_action"
   aliases:
@@ -28,5 +33,5 @@ _meta:
   managed: false
   # Bump this version to delete, reinstall, and restart the transform during
   # package installation.
-  fleet_transform_version: 1.0.0
+  fleet_transform_version: 1.0.1
   run_as_kibana_system: false
diff --git a/packages/microsoft_defender_endpoint/elasticsearch/transform/latest_cdr_vuln/transform.yml b/packages/microsoft_defender_endpoint/elasticsearch/transform/latest_cdr_vuln/transform.yml
@@ -9,6 +9,9 @@ source:
       filter:
         - exists:
             field: resource.id
+      must_not:
+        exists:
+          field: error.message
 dest:
   index: "security_solution-microsoft_defender_endpoint.vulnerability_latest-v2"
   aliases:
@@ -37,4 +40,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.2.0
+  fleet_transform_version: 0.2.1
diff --git a/packages/microsoft_defender_endpoint/manifest.yml b/packages/microsoft_defender_endpoint/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.4.0"
 name: microsoft_defender_endpoint
 title: Microsoft Defender for Endpoint
-version: "4.1.0"
+version: "4.1.1"
 description: Collect logs from Microsoft Defender for Endpoint with Elastic Agent.
 categories:
   - security
diff --git a/packages/qualys_vmdr/changelog.yml b/packages/qualys_vmdr/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "6.10.1"
+  changes:
+    - description: Update transform to filter out documents containing an error.message from vulnerability latest index
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15722
 - version: "6.10.0"
   changes:
     - description: Add `vulnerability_workflow` sub category label.
diff --git a/packages/qualys_vmdr/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml b/packages/qualys_vmdr/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml
@@ -1,6 +1,11 @@
 source:
   index:
     - "logs-qualys_vmdr.asset_host_detection-*"
+  query:
+    bool:
+      must_not:
+        exists:
+          field: error.message
 dest:
   index: "security_solution-qualys_vmdr.vulnerability_latest-v1"
   aliases:
@@ -27,4 +32,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.1.0
+  fleet_transform_version: 0.1.1
diff --git a/packages/qualys_vmdr/manifest.yml b/packages/qualys_vmdr/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.4.0"
 name: qualys_vmdr
 title: Qualys VMDR
-version: "6.10.0"
+version: "6.10.1"
 description: Collect data from Qualys VMDR platform with Elastic Agent.
 type: integration
 categories:
diff --git a/packages/rapid7_insightvm/changelog.yml b/packages/rapid7_insightvm/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.4.1"
+  changes:
+    - description: Update transform to filter out documents containing an error.message from vulnerability latest index
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15722
 - version: "2.4.0"
   changes:
     - description: Prevent updating fleet health status to degraded.
diff --git a/packages/rapid7_insightvm/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml b/packages/rapid7_insightvm/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml
@@ -1,6 +1,11 @@
 source:
   index:
     - "logs-rapid7_insightvm.asset_vulnerability-*"
+  query:
+    bool:
+      must_not:
+        exists:
+          field: error.message
 dest:
   index: "security_solution-rapid7_insightvm.vulnerability_latest-v1"
   aliases:
@@ -27,4 +32,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.1.0
+  fleet_transform_version: 0.1.1
diff --git a/packages/rapid7_insightvm/manifest.yml b/packages/rapid7_insightvm/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.4.0"
 name: rapid7_insightvm
 title: Rapid7 InsightVM
-version: "2.4.0"
+version: "2.4.1"
 source:
   license: "Elastic-2.0"
 description: Collect logs from Rapid7 InsightVM with Elastic Agent.
diff --git a/packages/tenable_io/changelog.yml b/packages/tenable_io/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "4.4.1"
+  changes:
+    - description: Update transform to filter out documents containing an error.message from vulnerability latest index
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15722
 - version: "4.4.0"
   changes:
     - description: Add `vulnerability_workflow` sub category label.
diff --git a/packages/tenable_io/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml b/packages/tenable_io/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml
diff --git a/packages/tenable_io/manifest.yml b/packages/tenable_io/manifest.yml
diff --git a/packages/wiz/changelog.yml b/packages/wiz/changelog.yml
diff --git a/packages/wiz/elasticsearch/transform/latest_cdr_misconfigurations/transform.yml b/packages/wiz/elasticsearch/transform/latest_cdr_misconfigurations/transform.yml
diff --git a/packages/wiz/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml b/packages/wiz/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml
diff --git a/packages/wiz/manifest.yml b/packages/wiz/manifest.yml
