Remove field instead of moving it

jsoriano · jsoriano · commit 326c7b10adbe · 2025-09-12T09:53:39.000+02:00
diff --git a/packages/elastic_agent/data_stream/auditbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/auditbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json
@@ -6,9 +6,7 @@
                     "libbeat": {
                         "pipeline": {
                             "queue": {
-                                "filled": {
-                                    "pct": 0.1
-                                }
+                                "filled": {}
                             }
                         }
                     }
diff --git a/packages/elastic_agent/data_stream/auditbeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/auditbeat_logs/elasticsearch/ingest_pipeline/default.yml
@@ -1,12 +1,11 @@
 ---
 description: "Pipeline for adapting Elastic Agent fields"
 processors:
-  - script:
-      lang: painless
-      description: Rename field from older versions of Beats.
-      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map && ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events != null
-      source: |
-          ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct = ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events;
+  - remove:
+      description: "Remove object that contained invalid data and can cause mapping conflicts"
+      field: monitoring.metrics.libbeat.pipeline.queue.filled.pct
+      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map
+      ignore_missing: true
 
 on_failure:
   - set:
diff --git a/packages/elastic_agent/data_stream/cloudbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/cloudbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json
@@ -6,9 +6,7 @@
                     "libbeat": {
                         "pipeline": {
                             "queue": {
-                                "filled": {
-                                    "pct": 0.1
-                                }
+                                "filled": {}
                             }
                         }
                     }
diff --git a/packages/elastic_agent/data_stream/cloudbeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/cloudbeat_logs/elasticsearch/ingest_pipeline/default.yml
@@ -1,12 +1,11 @@
 ---
 description: "Pipeline for adapting Elastic Agent fields"
 processors:
-  - script:
-      lang: painless
-      description: Rename field from older versions of Beats.
-      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map && ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events != null
-      source: |
-          ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct = ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events;
+  - remove:
+      description: "Remove object that contained invalid data and can cause mapping conflicts"
+      field: monitoring.metrics.libbeat.pipeline.queue.filled.pct
+      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map
+      ignore_missing: true
 
 on_failure:
   - set:
diff --git a/packages/elastic_agent/data_stream/filebeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/filebeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json
@@ -6,9 +6,7 @@
                     "libbeat": {
                         "pipeline": {
                             "queue": {
-                                "filled": {
-                                    "pct": 0.1
-                                }
+                                "filled": {}
                             }
                         }
                     }
diff --git a/packages/elastic_agent/data_stream/filebeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/filebeat_logs/elasticsearch/ingest_pipeline/default.yml
@@ -1,12 +1,11 @@
 ---
 description: "Pipeline for adapting Elastic Agent fields"
 processors:
-  - script:
-      lang: painless
-      description: Rename field from older versions of Beats.
-      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map && ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events != null
-      source: |
-          ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct = ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events;
+  - remove:
+      description: "Remove object that contained invalid data and can cause mapping conflicts"
+      field: monitoring.metrics.libbeat.pipeline.queue.filled.pct
+      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map
+      ignore_missing: true
 
 on_failure:
   - set:
diff --git a/packages/elastic_agent/data_stream/heartbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/heartbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json
@@ -6,9 +6,7 @@
                     "libbeat": {
                         "pipeline": {
                             "queue": {
-                                "filled": {
-                                    "pct": 0.1
-                                }
+                                "filled": {}
                             }
                         }
                     }
diff --git a/packages/elastic_agent/data_stream/heartbeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/heartbeat_logs/elasticsearch/ingest_pipeline/default.yml
@@ -1,12 +1,11 @@
 ---
 description: "Pipeline for adapting Elastic Agent fields"
 processors:
-  - script:
-      lang: painless
-      description: Rename field from older versions of Beats.
-      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map && ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events != null
-      source: |
-          ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct = ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events;
+  - remove:
+      description: "Remove object that contained invalid data and can cause mapping conflicts"
+      field: monitoring.metrics.libbeat.pipeline.queue.filled.pct
+      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map
+      ignore_missing: true
 
 on_failure:
   - set:
diff --git a/packages/elastic_agent/data_stream/metricbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/metricbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json
@@ -6,9 +6,7 @@
                     "libbeat": {
                         "pipeline": {
                             "queue": {
-                                "filled": {
-                                    "pct": 0.1
-                                }
+                                "filled": {}
                             }
                         }
                     }
diff --git a/packages/elastic_agent/data_stream/metricbeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/metricbeat_logs/elasticsearch/ingest_pipeline/default.yml
@@ -1,12 +1,11 @@
 ---
 description: "Pipeline for adapting Elastic Agent fields"
 processors:
-  - script:
-      lang: painless
-      description: Rename field from older versions of Beats.
-      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map && ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events != null
-      source: |
-          ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct = ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events;
+  - remove:
+      description: "Remove object that contained invalid data and can cause mapping conflicts"
+      field: monitoring.metrics.libbeat.pipeline.queue.filled.pct
+      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map
+      ignore_missing: true
 
 on_failure:
   - set:
diff --git a/packages/elastic_agent/data_stream/osquerybeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/osquerybeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json
@@ -6,9 +6,7 @@
                     "libbeat": {
                         "pipeline": {
                             "queue": {
-                                "filled": {
-                                    "pct": 0.1
-                                }
+                                "filled": {}
                             }
                         }
                     }
diff --git a/packages/elastic_agent/data_stream/osquerybeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/osquerybeat_logs/elasticsearch/ingest_pipeline/default.yml
@@ -1,12 +1,11 @@
 ---
 description: "Pipeline for adapting Elastic Agent fields"
 processors:
-  - script:
-      lang: painless
-      description: Rename field from older versions of Beats.
-      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map && ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events != null
-      source: |
-          ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct = ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events;
+  - remove:
+      description: "Remove object that contained invalid data and can cause mapping conflicts"
+      field: monitoring.metrics.libbeat.pipeline.queue.filled.pct
+      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map
+      ignore_missing: true
 
 on_failure:
   - set:
diff --git a/packages/elastic_agent/data_stream/packetbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/packetbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json
@@ -6,9 +6,7 @@
                     "libbeat": {
                         "pipeline": {
                             "queue": {
-                                "filled": {
-                                    "pct": 0.1
-                                }
+                                "filled": {}
                             }
                         }
                     }
diff --git a/packages/elastic_agent/data_stream/packetbeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/packetbeat_logs/elasticsearch/ingest_pipeline/default.yml
@@ -1,12 +1,11 @@
 ---
 description: "Pipeline for adapting Elastic Agent fields"
 processors:
-  - script:
-      lang: painless
-      description: Rename field from older versions of Beats.
-      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map && ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events != null
-      source: |
-          ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct = ctx.monitoring.metrics.libbeat.pipeline.queue.filled.pct.events;
+  - remove:
+      description: "Remove object that contained invalid data and can cause mapping conflicts"
+      field: monitoring.metrics.libbeat.pipeline.queue.filled.pct
+      if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map
+      ignore_missing: true
 
 on_failure:
   - set:

Original file line number	Diff line number	Diff line change
`@@ -6,9 +6,7 @@`
`6`	`6`	`"libbeat": {`
`7`	`7`	`"pipeline": {`
`8`	`8`	`"queue": {`
`9`		`- "filled": {`
`10`		`- "pct": 0.1`
`11`		`- }`
	`9`	`+ "filled": {}`
`12`	`10`	`}`
`13`	`11`	`}`
`14`	`12`	`}`