[panw] Generate processor tags and normalize error handler (#15565)

- Generate tags for processors missing tags
- Normalize the pipeline error handler

Author: taylor-swanson

packages/panw/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "5.3.5"
+  changes:
+    - description: Generate processor tags and normalize error handler.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15565
 - version: "5.3.4"
   changes:
     - description: Fix broken link on the Palo Alto Network Integration page.

packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/audit.yml

@@ -3,6 +3,7 @@ description: Pipeline for PanOS Audit Logs.
 # API Docs: https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/audit-log-fields
 processors:
   - csv:
+      tag: csv_message_4cf41818
       field: message
       ignore_failure: true
       target_fields:
@@ -12,23 +13,28 @@ processors:
         - panw.panos.cmd
         - event.outcome
   - uppercase:
+      tag: uppercase_panw_panos_type_ec638369
       field: panw.panos.type
       ignore_missing: true
-# Set event.kind & event.category field.
+  # Set event.kind & event.category field.
   - set:
+      tag: set_event_kind_de80643c
       field: event.kind
       value: event
   - append:
+      tag: append_event_category_c8ab9fa2
       field: event.category
       value:
         - configuration
 
-# Set custom fields to ECS fields
+  # Set custom fields to ECS fields
   - set:
+      tag: set_observer_hostname_5bc96c94
       field: observer.hostname
       copy_from: panw.panos.device_name
       ignore_failure: true
   - set:
+      tag: set_source_ip_c1b0f423
       field: source.ip
       copy_from: panw.panos.source.ip
       ignore_failure: true
@@ -40,7 +46,7 @@ on_failure:
   - append:
       field: error.message
       value: >-
-        error in Audit pipeline:
-        error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}}
-        with tag [{{{_ingest.on_failure_processor_tag}}}]{{{/_ingest.on_failure_processor_tag}}}
-        {{{_ingest.on_failure_message}}}
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'

packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/authentication.yml

@@ -3,6 +3,7 @@ description: Pipeline for PanOS Authentication Logs.
 # API Docs: https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/authentication-log-fields
 processors:
   - csv:
+      tag: csv_message_4c28b73f
       field: message
       ignore_failure: true
       target_fields:
@@ -47,50 +48,60 @@ processors:
         - panw.panos.flow_id
 
   - append:
+      tag: append_source_user_name_61f77e09
       field: source.user.name
       value: '{{{_temp_.user}}}'
       if: ctx._temp_?.user != null && ctx._temp_.user != ''
       allow_duplicates: false
       ignore_failure: true
   - append:
+      tag: append_source_user_name_7cf5bcf3
       field: source.user.name
       value: '{{{panw.panos.normalize_user}}}'
       if: ctx.panw?.panos?.normalize_user != null && ctx.panw.panos.normalize_user != ''
       allow_duplicates: false
       ignore_failure: true
 
-# Set event.kind & event.category field.
+  # Set event.kind & event.category field.
   - set:
+      tag: set_event_kind_de80643c
       field: event.kind
       value: event
   - append:
+      tag: append_event_category_c182a53a
       field: event.category
       value:
         - authentication
 
-# Set event.outcome
+  # Set event.outcome
   - set:
+      tag: set_event_outcome_1af271cd
       field: event.outcome
       value: success
 
-# Set custom fields to ECS fields
+  # Set custom fields to ECS fields
   - set:
+      tag: set_observer_hostname_5bc96c94
       field: observer.hostname
       copy_from: panw.panos.device_name
       ignore_failure: true
   - set:
+      tag: set_rule_uuid_5478ecb2
       field: rule.uuid
       copy_from: panw.panos.rule_uuid
       ignore_failure: true
   - set:
+      tag: set_source_ip_c1b0f423
       field: source.ip
       copy_from: panw.panos.source.ip
       ignore_failure: true
   - set:
+      tag: set_panw_panos_user_007bc289
       field: panw.panos.user
       copy_from: _temp_.user
       ignore_failure: true
   - set:
+      tag: set_panw_panos_user_agent_f4b744c1
       field: panw.panos.user_agent
       copy_from: _temp_.user_agent
       ignore_failure: true
@@ -101,7 +112,7 @@ on_failure:
   - append:
       field: error.message
       value: >-
-        error in Authentication pipeline:
-        error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}}
-        with tag [{{{_ingest.on_failure_processor_tag}}}]{{{/_ingest.on_failure_processor_tag}}}
-        {{{_ingest.on_failure_message}}}
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'

packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/config.yml

@@ -3,6 +3,7 @@ description: Pipeline for PanOS Config Logs.
 # API Docs: https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/config-log-fields
 processors:
   - csv:
+      tag: csv_message_e2e3e7de
       field: message
       target_fields:
         - panw.panos.host.ip
@@ -14,6 +15,7 @@ processors:
         - panw.panos.path
         - _temp_.check_field
   - convert:
+      tag: convert__temp__check_field_5a04d649
       field: _temp_.check_field
       type: long
       ignore_missing: true
@@ -71,6 +73,7 @@ processors:
         - _temp_.future_use1
         - _temp_.high_res_timestamp
   - script:
+      tag: script_edc601fd
       description: Set Event Action.
       lang: painless
       ignore_failure: true
@@ -84,53 +87,62 @@ processors:
         move: cmd-move
         rename: cmd-rename
         set: cmd-set
-      source:
-        ctx.event.action = params.get(ctx.panw.panos.cmd);
+      source: ctx.event.action = params.get(ctx.panw.panos.cmd);
   - set:
+      tag: set_event_outcome_a8d719f3
       field: event.outcome
       value: success
       ignore_failure: true
       if: ctx.panw?.panos?.result == 'Succeeded'
   - set:
+      tag: set_event_outcome_98bc1180
       field: event.outcome
       value: failure
       ignore_failure: true
       if: ctx.panw?.panos?.result == 'Failed'
   - set:
+      tag: set_event_outcome_332ce513
       field: event.outcome
       value: unknown
       ignore_failure: true
       if: '!["Succeeded", "Failed"].contains(ctx.panw.panos.action)'
 
-# Set event.kind & event.category field.
+  # Set event.kind & event.category field.
   - set:
+      tag: set_event_kind_de80643c
       field: event.kind
       value: event
   - append:
+      tag: append_event_category_c8ab9fa2
       field: event.category
       value:
         - configuration
 
-# Set custom fields to ECS fields
+  # Set custom fields to ECS fields
   - set:
+      tag: set_host_ip_ae377e8b
       field: host.ip
       copy_from: panw.panos.host.ip
       ignore_failure: true
   - set:
+      tag: set_observer_hostname_5bc96c94
       field: observer.hostname
       copy_from: panw.panos.device_name
       ignore_failure: true
 
-# Set event.outcome from panw.panos.result: documented allowed values are Submitted, Succeeded, Failed, and Unauthorized
+  # Set event.outcome from panw.panos.result: documented allowed values are Submitted, Succeeded, Failed, and Unauthorized
   - set:
+      tag: set_event_outcome_057c7750
       if: ctx.panw?.panos?.result == "Succeeded"
       field: event.outcome
       value: success
   - set:
+      tag: set_event_outcome_777b32fd
       if: ctx.panw?.panos?.result == "Failed"
       field: event.outcome
       value: failure
   - set:
+      tag: set_event_outcome_bcf074fb
       if: ctx.event?.outcome == null || ctx.event.outcome == ""
       field: event.outcome
       value: unknown
@@ -142,7 +154,7 @@ on_failure:
   - append:
       field: error.message
       value: >-
-        error in Config pipeline:
-        error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}}
-        with tag [{{{_ingest.on_failure_processor_tag}}}]{{{/_ingest.on_failure_processor_tag}}}
-        {{{_ingest.on_failure_message}}}
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'

packages/panw/data_stream/panos/elasticsearch/ingest_pipeline/correlated_event.yml

@@ -3,6 +3,7 @@ description: Pipeline for PanOS Correlated Event Logs.
 # API Docs: https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/correlated-events-log-fields
 processors:
   - csv:
+      tag: csv_message_9f7a7cd8
       field: message
       ignore_failure: true
       target_fields:
@@ -22,30 +23,36 @@ processors:
         - panw.panos.object.id
         - panw.panos.evidence
 
-# Set event.kind & event.category field.
+  # Set event.kind & event.category field.
   - set:
+      tag: set_event_kind_de80643c
       field: event.kind
       value: event
   - append:
+      tag: append_event_category_38e4ac56
       field: event.category
       value:
         - network
 
-# Set event.outcome
+  # Set event.outcome
   - set:
+      tag: set_event_outcome_1af271cd
       field: event.outcome
       value: success
 
-# Set custom fields to ECS fields
+  # Set custom fields to ECS fields
   - set:
+      tag: set_log_level_ed3c9f74
       field: log.level
       copy_from: panw.panos.severity
       ignore_failure: true
   - set:
+      tag: set_observer_hostname_5bc96c94
       field: observer.hostname
       copy_from: panw.panos.device_name
       ignore_failure: true
   - set:
+      tag: set_source_ip_c1b0f423
       field: source.ip
       copy_from: panw.panos.source.ip
       ignore_failure: true
@@ -57,7 +64,7 @@ on_failure:
   - append:
       field: error.message
       value: >-
-        error in Correlated Event pipeline:
-        error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}}
-        with tag [{{{_ingest.on_failure_processor_tag}}}]{{{/_ingest.on_failure_processor_tag}}}
-        {{{_ingest.on_failure_message}}}
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'