feat(citrix_adc): Preserve event.original when errors occur in pipelines in log data stream (#15902)

- Added append processor to global on_failure to preserve event original
- Added append processor to default pipelines to preserve event original if error.message is set

Author: taylor-swanson

packages/citrix_adc/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.18.0"
+  changes:
+    - description: Preserve event.original on pipeline error in log data stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15902
 - version: "1.17.5"
   changes:
     - description: Properly parse failed status conditions in sslvpn pipeline

packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/alg_feature.yml

@@ -197,3 +197,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/appfw_feature.yml

@@ -194,3 +194,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/bot_feature.yml

@@ -40,3 +40,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/cef.yml

@@ -135,3 +135,7 @@ on_failure:
   - set:
       field: event.kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/ci_feature.yml

@@ -241,3 +241,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/cvpn_feature.yml

@@ -37,3 +37,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -55,8 +55,8 @@ processors:
       type: long
       ignore_missing: true
 
-# Time zone (in order: log, config, locale, default to UTC).
 
+  # Time zone (in order: log, config, locale, default to UTC).
   - set:
       field: _tmp.tz
       value: UTC
@@ -79,8 +79,8 @@ processors:
       field: event.timezone
       copy_from: _tmp.tz
 
-# Syslog timestamp
 
+  # Syslog timestamp
   - date:
       if: ctx._tmp?.timestamp8601 != null
       tag: date_syslog_timestamp8601
@@ -174,8 +174,8 @@ processors:
           }
         });
 
-# Native-format timestamp
 
+  # Native-format timestamp
   - date:
       tag: date_timestamp_native
       field: _tmp.timestamp_native
@@ -257,8 +257,8 @@ processors:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
 
-# Move vendor time fields to ECS.
 
+  # Move vendor time fields to ECS.
   - set:
       tag: set_@timestamp_from_citrix_native
       field: '@timestamp'
@@ -450,6 +450,12 @@ processors:
         - _conf
       tag: remove_tmp_and_conf
       ignore_missing: true
+  - append:
+      tag: append_preserve_original_event_on_error
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false
+      if: ctx.error?.message != null
 on_failure:
   - remove:
       field:
@@ -465,3 +471,7 @@ on_failure:
       field: event.kind
       tag: set_pipeline_error_to_event_kind
       value: pipeline_error
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/dns_and_ssli_feature.yml

@@ -112,3 +112,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false

packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/ica_feature.yml

@@ -584,3 +584,7 @@ on_failure:
         {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
         {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
         failed with message '{{{ _ingest.on_failure_message }}}'
+  - append:
+      field: tags
+      value: preserve_original_event
+      allow_duplicates: false