From c1a52e3242bcc23396e9bc434d761b26169960ed Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Wed, 29 Oct 2025 12:32:24 -0500 Subject: [PATCH] feat: Preserve event.original when errors occur in pipelines - Added append processor to global on_failure to preserve event original - Added append processor to default pipelines to preserve event original if error.message is set Affects the following integrations: - imperva - iptables - juniper_srx - modsecurity - netflow - panw - pfsense - proxysg - qnap_nas - snort - sonicwall_firewall --- packages/imperva/changelog.yml | 5 +++++ .../elasticsearch/ingest_pipeline/default.yml | 10 ++++++++++ packages/imperva/manifest.yml | 2 +- packages/iptables/changelog.yml | 5 +++++ .../log/elasticsearch/ingest_pipeline/default.yml | 10 ++++++++++ packages/iptables/manifest.yml | 2 +- packages/juniper_srx/changelog.yml | 5 +++++ .../log/elasticsearch/ingest_pipeline/atp.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/default.yml | 10 ++++++++++ .../log/elasticsearch/ingest_pipeline/flow.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/idp.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/ids.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/secintel.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/system.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/utm.yml | 4 ++++ packages/juniper_srx/manifest.yml | 2 +- packages/modsecurity/changelog.yml | 5 +++++ .../elasticsearch/ingest_pipeline/apache-modsec.yml | 4 ++++ .../auditlog/elasticsearch/ingest_pipeline/default.yml | 10 ++++++++++ .../elasticsearch/ingest_pipeline/nginx-modsec.yml | 4 ++++ packages/modsecurity/manifest.yml | 2 +- packages/netflow/changelog.yml | 5 +++++ .../log/elasticsearch/ingest_pipeline/default.yml | 10 ++++++++++ packages/netflow/manifest.yml | 2 +- packages/panw/changelog.yml | 5 +++++ .../panos/elasticsearch/ingest_pipeline/audit.yml | 4 ++++ .../elasticsearch/ingest_pipeline/authentication.yml | 4 ++++ .../panos/elasticsearch/ingest_pipeline/config.yml | 4 ++++ .../elasticsearch/ingest_pipeline/correlated_event.yml | 4 ++++ .../panos/elasticsearch/ingest_pipeline/decryption.yml | 4 ++++ .../panos/elasticsearch/ingest_pipeline/default.yml | 10 ++++++++++ .../elasticsearch/ingest_pipeline/globalprotect.yml | 4 ++++ .../panos/elasticsearch/ingest_pipeline/gtp.yml | 4 ++++ .../panos/elasticsearch/ingest_pipeline/hipmatch.yml | 4 ++++ .../panos/elasticsearch/ingest_pipeline/ip_tag.yml | 4 ++++ .../panos/elasticsearch/ingest_pipeline/sctp.yml | 4 ++++ .../panos/elasticsearch/ingest_pipeline/system.yml | 4 ++++ .../panos/elasticsearch/ingest_pipeline/threat.yml | 4 ++++ .../panos/elasticsearch/ingest_pipeline/traffic.yml | 4 ++++ .../ingest_pipeline/tunnel_inspection.yml | 4 ++++ .../panos/elasticsearch/ingest_pipeline/userid.yml | 4 ++++ packages/panw/manifest.yml | 2 +- packages/pfsense/changelog.yml | 5 +++++ .../log/elasticsearch/ingest_pipeline/default.yml | 10 ++++++++++ .../log/elasticsearch/ingest_pipeline/dhcp.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/firewall.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/haproxy.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/ipsec.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/openvpn.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/php-fpm.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/snort.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/squid.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/unbound.yml | 4 ++++ packages/pfsense/manifest.yml | 2 +- packages/proxysg/changelog.yml | 5 +++++ .../ingest_pipeline/bcreportermain_v1.yml | 4 ++++ .../elasticsearch/ingest_pipeline/bcreporterssl_v1.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/default.yml | 10 ++++++++++ .../log/elasticsearch/ingest_pipeline/main.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/ssl.yml | 4 ++++ packages/proxysg/manifest.yml | 2 +- packages/qnap_nas/changelog.yml | 5 +++++ .../log/elasticsearch/ingest_pipeline/default.yml | 10 ++++++++++ packages/qnap_nas/manifest.yml | 2 +- packages/snort/changelog.yml | 5 +++++ .../log/elasticsearch/ingest_pipeline/default.yml | 10 ++++++++++ .../log/elasticsearch/ingest_pipeline/json.yml | 4 ++++ .../log/elasticsearch/ingest_pipeline/plaintext.yml | 4 ++++ packages/snort/manifest.yml | 2 +- packages/sonicwall_firewall/changelog.yml | 5 +++++ .../log/elasticsearch/ingest_pipeline/default.yml | 10 ++++++++++ packages/sonicwall_firewall/manifest.yml | 2 +- 72 files changed, 332 insertions(+), 11 deletions(-) diff --git a/packages/imperva/changelog.yml b/packages/imperva/changelog.yml index 42f7ee51232..1ba5d856d89 100644 --- a/packages/imperva/changelog.yml +++ b/packages/imperva/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.9.0" + changes: + - description: Preserve event.original on pipeline error. + type: enhancement + link: https://github.com/elastic/integrations/pull/15805 - version: "1.8.2" changes: - description: Generate processor tags and normalize error handler. diff --git a/packages/imperva/data_stream/securesphere/elasticsearch/ingest_pipeline/default.yml b/packages/imperva/data_stream/securesphere/elasticsearch/ingest_pipeline/default.yml index f2014b83b5c..b2314199622 100644 --- a/packages/imperva/data_stream/securesphere/elasticsearch/ingest_pipeline/default.yml +++ b/packages/imperva/data_stream/securesphere/elasticsearch/ingest_pipeline/default.yml @@ -336,6 +336,12 @@ processors: tag: set_pipeline_error_to_event_kind value: pipeline_error if: ctx.error?.message != null + - append: + tag: append_preserve_original_event_on_error + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message @@ -347,3 +353,7 @@ on_failure: - set: field: event.kind value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/imperva/manifest.yml b/packages/imperva/manifest.yml index e0def3edf42..2adb102ba00 100644 --- a/packages/imperva/manifest.yml +++ b/packages/imperva/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: imperva title: Imperva -version: "1.8.2" +version: "1.9.0" description: Collect logs from Imperva devices with Elastic Agent. categories: ["network", "security"] type: integration diff --git a/packages/iptables/changelog.yml b/packages/iptables/changelog.yml index 64967cb27ea..ef9cdf01035 100644 --- a/packages/iptables/changelog.yml +++ b/packages/iptables/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.22.0" + changes: + - description: Preserve event.original on pipeline error. + type: enhancement + link: https://github.com/elastic/integrations/pull/15805 - version: "1.21.4" changes: - description: Generate processor tags and normalize error handler. diff --git a/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 2c591e0d36f..c327a7d0154 100644 --- a/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -415,6 +415,12 @@ processors: field: - _tmp ignore_failure: true + - append: + tag: append_preserve_original_event_on_error + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - remove: field: @@ -430,3 +436,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/iptables/manifest.yml b/packages/iptables/manifest.yml index 298d06a19e8..1cb9aff8cd2 100644 --- a/packages/iptables/manifest.yml +++ b/packages/iptables/manifest.yml @@ -1,6 +1,6 @@ name: iptables title: Iptables -version: "1.21.4" +version: "1.22.0" description: Collect logs from Iptables with Elastic Agent. type: integration icons: diff --git a/packages/juniper_srx/changelog.yml b/packages/juniper_srx/changelog.yml index cf188583c70..e0cabed6fb8 100644 --- a/packages/juniper_srx/changelog.yml +++ b/packages/juniper_srx/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.26.0" + changes: + - description: Preserve event.original on pipeline error. + type: enhancement + link: https://github.com/elastic/integrations/pull/15805 - version: "1.25.2" changes: - description: Generate processor tags and normalize error handler. diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/atp.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/atp.yml index 97bdf018e97..39570e143fd 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/atp.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/atp.yml @@ -427,3 +427,7 @@ on_failure: - set: field: event.kind value: pipeline_error +- append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml index a3f1b6d8818..a6809113738 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -434,6 +434,12 @@ processors: - juniper.srx.dstzone - syslog_pri ignore_missing: true + - append: + tag: append_preserve_original_event_on_error + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: field: error.message @@ -445,3 +451,7 @@ on_failure: - set: field: event.kind value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/flow.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/flow.yml index ca0bd5ecb0f..352c3e6e3bd 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/flow.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/flow.yml @@ -427,3 +427,7 @@ on_failure: - set: field: event.kind value: pipeline_error +- append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/idp.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/idp.yml index 0023752c11d..99c59615d1b 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/idp.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/idp.yml @@ -338,3 +338,7 @@ on_failure: - set: field: event.kind value: pipeline_error +- append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/ids.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/ids.yml index 4e5c967fef2..31e18350c7f 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/ids.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/ids.yml @@ -429,3 +429,7 @@ on_failure: - set: field: event.kind value: pipeline_error +- append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/secintel.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/secintel.yml index 09fef816ef7..1f282f2baea 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/secintel.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/secintel.yml @@ -411,3 +411,7 @@ on_failure: - set: field: event.kind value: pipeline_error +- append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml index c6f3549dacc..4e2718dd12a 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml @@ -941,3 +941,7 @@ on_failure: - set: field: event.kind value: pipeline_error +- append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/utm.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/utm.yml index 764149fea9e..478e7d842b1 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/utm.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/utm.yml @@ -458,3 +458,7 @@ on_failure: - set: field: event.kind value: pipeline_error +- append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/juniper_srx/manifest.yml b/packages/juniper_srx/manifest.yml index 25acd08cf21..99f317820ed 100644 --- a/packages/juniper_srx/manifest.yml +++ b/packages/juniper_srx/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: juniper_srx title: Juniper SRX -version: "1.25.2" +version: "1.26.0" description: Collect logs from Juniper SRX devices with Elastic Agent. categories: ["network", "security", "firewall_security"] type: integration diff --git a/packages/modsecurity/changelog.yml b/packages/modsecurity/changelog.yml index 89829f77270..5e05c27ba90 100644 --- a/packages/modsecurity/changelog.yml +++ b/packages/modsecurity/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.22.0" + changes: + - description: Preserve event.original on pipeline error. + type: enhancement + link: https://github.com/elastic/integrations/pull/15805 - version: "1.21.2" changes: - description: Generate processor tags and normalize error handler. diff --git a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/apache-modsec.yml b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/apache-modsec.yml index bc6708ae69a..3e14ac4871e 100644 --- a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/apache-modsec.yml +++ b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/apache-modsec.yml @@ -300,3 +300,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/default.yml b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/default.yml index 9cbe19202bd..84f365ce78b 100644 --- a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/default.yml +++ b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/default.yml @@ -41,6 +41,12 @@ processors: tag: pipeline_7206172d name: '{{ IngestPipeline "apache-modsec" }}' if: (ctx.modsec?.audit?.server != null && ctx.modsec.audit.server.toLowerCase().contains('apache')) || (ctx.modsec?.audit?.connector != null && ctx.modsec.audit.connector.toLowerCase().contains('apache')) + - append: + tag: append_preserve_original_event_on_error + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - set: field: event.kind @@ -52,3 +58,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/nginx-modsec.yml b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/nginx-modsec.yml index f1e606b059d..5d38da9015d 100644 --- a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/nginx-modsec.yml +++ b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/nginx-modsec.yml @@ -210,3 +210,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/modsecurity/manifest.yml b/packages/modsecurity/manifest.yml index c6b241599b6..fa88d44fe67 100644 --- a/packages/modsecurity/manifest.yml +++ b/packages/modsecurity/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: modsecurity title: "ModSecurity Audit" -version: "1.21.2" +version: "1.22.0" description: Collect logs from ModSecurity with Elastic Agent type: integration categories: diff --git a/packages/netflow/changelog.yml b/packages/netflow/changelog.yml index f1610023c6e..5b2a65e30c1 100644 --- a/packages/netflow/changelog.yml +++ b/packages/netflow/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.24.0" + changes: + - description: Preserve event.original on pipeline error. + type: enhancement + link: https://github.com/elastic/integrations/pull/15805 - version: "2.23.2" changes: - description: Generate processor tags and normalize error handler. diff --git a/packages/netflow/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/netflow/data_stream/log/elasticsearch/ingest_pipeline/default.yml index ba3343ed5c1..72a679713c2 100644 --- a/packages/netflow/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/netflow/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -188,6 +188,12 @@ processors: field: - _tmp_ ignore_missing: true + - append: + tag: append_preserve_original_event_on_error + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - set: field: event.kind @@ -199,3 +205,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/netflow/manifest.yml b/packages/netflow/manifest.yml index ddde84c9df8..cebe14f4549 100644 --- a/packages/netflow/manifest.yml +++ b/packages/netflow/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: netflow title: NetFlow Records -version: "2.23.2" +version: "2.24.0" description: Collect flow records from NetFlow and IPFIX exporters with Elastic Agent. type: integration categories: diff --git a/packages/panw/changelog.yml b/packages/panw/changelog.yml index 1067f852b8c..e870196f957 100644 --- a/packages/panw/changelog.yml +++ b/packages/panw/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "5.4.0" + changes: + - description: Preserve event.original on pipeline error. + type: enhancement + link: https://github.com/elastic/integrations/pull/15805 - version: "5.3.5" changes: - description: Generate processor tags and normalize error handler. diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/audit.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/audit.yml index 70c8622584a..629b7b8bd16 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/audit.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/audit.yml @@ -50,3 +50,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/authentication.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/authentication.yml index e5348184e79..d2fd877d271 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/authentication.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/authentication.yml @@ -116,3 +116,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/config.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/config.yml index 059d2ba95bf..a39085205b0 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/config.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/config.yml @@ -158,3 +158,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/correlated_event.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/correlated_event.yml index 26fe3f9e2d9..a0783256c96 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/correlated_event.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/correlated_event.yml @@ -68,3 +68,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/decryption.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/decryption.yml index ec4cac9011d..c7757fc355f 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/decryption.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/decryption.yml @@ -386,3 +386,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/default.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/default.yml index 141653dd40d..adf68ed3f02 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/default.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/default.yml @@ -2057,6 +2057,12 @@ processors: return false; } dropEmptyFields(ctx); + - append: + tag: append_preserve_original_event_on_error + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - set: @@ -2075,3 +2081,7 @@ on_failure: - _conf - message ignore_missing: true + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/globalprotect.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/globalprotect.yml index 8824052cf2b..b1b478bc016 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/globalprotect.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/globalprotect.yml @@ -164,3 +164,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/gtp.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/gtp.yml index 033c0b40868..79ba195616f 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/gtp.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/gtp.yml @@ -205,3 +205,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/hipmatch.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/hipmatch.yml index 4bdba3ebab4..3b01953c111 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/hipmatch.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/hipmatch.yml @@ -96,3 +96,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/ip_tag.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/ip_tag.yml index 6a65d70e3dc..d70690f66d7 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/ip_tag.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/ip_tag.yml @@ -66,3 +66,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/sctp.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/sctp.yml index e14a02909b9..5cb4ac0b8d2 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/sctp.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/sctp.yml @@ -170,3 +170,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/system.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/system.yml index a0a3f409dfc..79541b341dd 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/system.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/system.yml @@ -66,3 +66,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/threat.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/threat.yml index 8a859f74c56..8c45af8d181 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/threat.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/threat.yml @@ -470,3 +470,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/traffic.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/traffic.yml index a680b67303e..d57b8e03396 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/traffic.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/traffic.yml @@ -381,3 +381,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/tunnel_inspection.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/tunnel_inspection.yml index fbca0d7ee88..ab32774810e 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/tunnel_inspection.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/tunnel_inspection.yml @@ -261,3 +261,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/userid.yml b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/userid.yml index bac4b5ef7ce..45ff6b684d3 100644 --- a/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/userid.yml +++ b/packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/userid.yml @@ -107,3 +107,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/panw/manifest.yml b/packages/panw/manifest.yml index 727acc52c25..587fe4df7e7 100644 --- a/packages/panw/manifest.yml +++ b/packages/panw/manifest.yml @@ -1,6 +1,6 @@ name: panw title: Palo Alto Next-Gen Firewall -version: "5.3.5" +version: "5.4.0" description: Collect logs from Palo Alto next-gen firewalls with Elastic Agent. type: integration format_version: "3.0.3" diff --git a/packages/pfsense/changelog.yml b/packages/pfsense/changelog.yml index a8462ff8de6..039cd01dfab 100644 --- a/packages/pfsense/changelog.yml +++ b/packages/pfsense/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.24.0" + changes: + - description: Preserve event.original on pipeline error. + type: enhancement + link: https://github.com/elastic/integrations/pull/15805 - version: "1.23.2" changes: - description: Generate processor tags and normalize error handler. diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 3ffb00e757d..d31aed61322 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -267,6 +267,12 @@ processors: } } handleMap(ctx); + - append: + tag: append_preserve_original_event_on_error + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - remove: field: @@ -282,3 +288,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml index 21b4d1e988e..c1512b65b03 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml @@ -114,3 +114,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml index 88f6ae815e4..14d129b64d4 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/firewall.yml @@ -104,3 +104,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml index c339a312c36..df4c4e9defd 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/haproxy.yml @@ -118,3 +118,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/ipsec.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/ipsec.yml index 522a876c389..bfa60e05965 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/ipsec.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/ipsec.yml @@ -48,3 +48,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/openvpn.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/openvpn.yml index 758502d2809..1452a3adb30 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/openvpn.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/openvpn.yml @@ -63,3 +63,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/php-fpm.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/php-fpm.yml index e9c16be1131..05533617511 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/php-fpm.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/php-fpm.yml @@ -55,3 +55,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/snort.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/snort.yml index 23d4542b62a..a55e74a9fd4 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/snort.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/snort.yml @@ -21,3 +21,7 @@ on_failure: - set: field: event.kind value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/squid.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/squid.yml index 92dc497f0df..04cce5174ad 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/squid.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/squid.yml @@ -42,3 +42,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/unbound.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/unbound.yml index 4d6112e4554..c3e25159206 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/unbound.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/unbound.yml @@ -74,3 +74,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/pfsense/manifest.yml b/packages/pfsense/manifest.yml index 595f1efae25..ab3ef4d2653 100644 --- a/packages/pfsense/manifest.yml +++ b/packages/pfsense/manifest.yml @@ -1,6 +1,6 @@ name: pfsense title: pfSense -version: "1.23.2" +version: "1.24.0" description: Collect logs from pfSense and OPNsense with Elastic Agent. type: integration icons: diff --git a/packages/proxysg/changelog.yml b/packages/proxysg/changelog.yml index cba2da12a06..c5517a29e8f 100644 --- a/packages/proxysg/changelog.yml +++ b/packages/proxysg/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.7.0" + changes: + - description: Preserve event.original on pipeline error. + type: enhancement + link: https://github.com/elastic/integrations/pull/15805 - version: "0.6.3" changes: - description: Generate processor tags and normalize error handler. diff --git a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/bcreportermain_v1.yml b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/bcreportermain_v1.yml index aa399c5f347..69396a03cda 100644 --- a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/bcreportermain_v1.yml +++ b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/bcreportermain_v1.yml @@ -67,3 +67,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/bcreporterssl_v1.yml b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/bcreporterssl_v1.yml index 35f4cd825ed..5ec8700cae6 100644 --- a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/bcreporterssl_v1.yml +++ b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/bcreporterssl_v1.yml @@ -58,3 +58,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/default.yml index cd0e00a08c6..8e9435d704c 100644 --- a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -279,6 +279,12 @@ processors: value: "{{{client.user.name}}}" if: ctx.source?.ip != null allow_duplicates: false + - append: + tag: append_preserve_original_event_on_error + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - append: @@ -294,3 +300,7 @@ on_failure: - remove: field: "_temp_" ignore_failure: true + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/main.yml b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/main.yml index 44851d6f4a4..95bf0a37d4c 100644 --- a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/main.yml +++ b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/main.yml @@ -42,3 +42,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/ssl.yml b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/ssl.yml index c0dee7df173..2b2f20c7e08 100644 --- a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/ssl.yml +++ b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/ssl.yml @@ -40,3 +40,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/proxysg/manifest.yml b/packages/proxysg/manifest.yml index 664445b4873..a051075ae71 100644 --- a/packages/proxysg/manifest.yml +++ b/packages/proxysg/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.1 name: proxysg title: "Broadcom ProxySG" -version: "0.6.3" +version: "0.7.0" source: license: "Elastic-2.0" description: "Collect access logs from Broadcom ProxySG with Elastic Agent." diff --git a/packages/qnap_nas/changelog.yml b/packages/qnap_nas/changelog.yml index e05facf858b..57b05073567 100644 --- a/packages/qnap_nas/changelog.yml +++ b/packages/qnap_nas/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.24.0" + changes: + - description: Preserve event.original on pipeline error. + type: enhancement + link: https://github.com/elastic/integrations/pull/15805 - version: "1.23.2" changes: - description: Generate processor tags and normalize error handler. diff --git a/packages/qnap_nas/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/qnap_nas/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 70b4ed73cbd..eea1ffd6d7f 100644 --- a/packages/qnap_nas/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/qnap_nas/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -313,6 +313,12 @@ processors: } } handleMap(ctx); + - append: + tag: append_preserve_original_event_on_error + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - set: field: event.kind @@ -324,3 +330,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/qnap_nas/manifest.yml b/packages/qnap_nas/manifest.yml index e2f01457d4e..ced2e3aa67c 100644 --- a/packages/qnap_nas/manifest.yml +++ b/packages/qnap_nas/manifest.yml @@ -1,6 +1,6 @@ name: qnap_nas title: QNAP NAS -version: "1.23.2" +version: "1.24.0" description: Collect logs from QNAP NAS devices with Elastic Agent. type: integration format_version: "3.0.3" diff --git a/packages/snort/changelog.yml b/packages/snort/changelog.yml index 5a454e7255a..a9e718cccee 100644 --- a/packages/snort/changelog.yml +++ b/packages/snort/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.20.0" + changes: + - description: Preserve event.original on pipeline error. + type: enhancement + link: https://github.com/elastic/integrations/pull/15805 - version: "1.19.2" changes: - description: Generate processor tags and normalize error handler. diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 8dd7942b526..ecd048e454c 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -233,6 +233,12 @@ processors: - _tmp - json ignore_missing: true + - append: + tag: append_preserve_original_event_on_error + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - set: field: event.kind @@ -244,3 +250,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml index d1f67f936e7..788f1ca85a0 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/json.yml @@ -217,3 +217,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml index f87dba5ebaf..197aa964099 100644 --- a/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml +++ b/packages/snort/data_stream/log/elasticsearch/ingest_pipeline/plaintext.yml @@ -60,3 +60,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/snort/manifest.yml b/packages/snort/manifest.yml index 7928819fed2..d4b380e4e9a 100644 --- a/packages/snort/manifest.yml +++ b/packages/snort/manifest.yml @@ -1,6 +1,6 @@ name: snort title: Snort -version: "1.19.2" +version: "1.20.0" description: Collect logs from Snort with Elastic Agent. type: integration icons: diff --git a/packages/sonicwall_firewall/changelog.yml b/packages/sonicwall_firewall/changelog.yml index 414aef5d2d6..121373617a2 100644 --- a/packages/sonicwall_firewall/changelog.yml +++ b/packages/sonicwall_firewall/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.20.0" + changes: + - description: Preserve event.original on pipeline error. + type: enhancement + link: https://github.com/elastic/integrations/pull/15805 - version: "1.19.2" changes: - description: Generate processor tags and normalize error handler. diff --git a/packages/sonicwall_firewall/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/sonicwall_firewall/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 186cd52c79d..4346117d7ea 100644 --- a/packages/sonicwall_firewall/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sonicwall_firewall/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -1403,6 +1403,12 @@ processors: tag: remove_sonicwall_288fe141 field: sonicwall if: 'ctx.sonicwall?.firewall?.size() == 0' + - append: + tag: append_preserve_original_event_on_error + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - set: @@ -1415,3 +1421,7 @@ on_failure: {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/sonicwall_firewall/manifest.yml b/packages/sonicwall_firewall/manifest.yml index ef21cc8cac0..da5ac5fddbb 100644 --- a/packages/sonicwall_firewall/manifest.yml +++ b/packages/sonicwall_firewall/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: sonicwall_firewall title: "SonicWall Firewall" -version: "1.19.2" +version: "1.20.0" description: "Integration for SonicWall firewall logs" type: integration categories: