[cisco_secure_email_gateway] Generate processor tags and normalize error handler (#15534)

- Generate tags for processors missing tags
- Normalize the pipeline error handler

Author: taylor-swanson

packages/cisco_secure_email_gateway/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.27.2"
+  changes:
+    - description: Generate processor tags and normalize error handler.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15534
 - version: "1.27.1"
   changes:
     - description: Changed owners.

packages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -2,54 +2,66 @@
 description: Pipeline for Cisco Secure Email Gateway logs.
 processors:
   - set:
+      tag: set_ecs_version_f5923549
       field: ecs.version
       value: '8.17.0'
   - set:
+      tag: set__tmp_filepath_bc60fe22
       field: _tmp.filepath
       value: '{{{log.file.path}}}'
       if: ctx.log?.file?.path != null
   - grok:
+      tag: grok__tmp_filepath_3fc3092e
       field: _tmp.filepath
       if: ctx.log?.file?.path != null
       ignore_failure: true
       patterns:
         - '^%{DATA}[\\/]%{WORD:cisco_secure_email_gateway.log.category.name}(?:\.%{HOSTNAME:cisco_secure_email_gateway.log.host})?\.@%{GREEDYDATA}\.s$'
   - rename:
+      tag: rename_message_to_event_original_56a77271
       field: message
       target_field: event.original
       ignore_missing: true
       if: ctx.event?.original == null
   - grok:
+      tag: grok_event_original_0ef2f0a7
       field: event.original
       patterns:
         - '^(?:<%{NUMBER:log.syslog.priority:long}>)?%{SYSLOGTIMESTAMP:_tmp.timestamp} (?:%{HOSTNAME:cisco_secure_email_gateway.log.host} )?%{NOTSPACE:cisco_secure_email_gateway.log.category.name}: %{WORD:log.level}: %{GREEDYDATA:cisco_secure_email_gateway.log.message}$'
         - '^(?:<%{NUMBER:log.syslog.priority:long}>)?%{SYSLOGTIMESTAMP:_tmp.timestamp} (?:%{HOSTNAME:cisco_secure_email_gateway.log.host} )?%{NOTSPACE:cisco_secure_email_gateway.log.category.name}: %{GREEDYDATA:cisco_secure_email_gateway.log.message}$'
         - '^%{DATA:_tmp.timestamp} %{WORD:log.level}: %{GREEDYDATA:cisco_secure_email_gateway.log.message}$'
         - '^%{GREEDYDATA:cisco_secure_email_gateway.log.message}$'
   - trim:
+      tag: trim_cisco_secure_email_gateway_log_message_610465ba
       field: cisco_secure_email_gateway.log.message
       ignore_failure: true
   - lowercase:
+      tag: lowercase_log_level_e59b6b49
       field: log.level
       ignore_failure: true
   # Time zone can come from two sources, choose in order: config, locale, default to UTC.
   - set:
+      tag: set__tmp_tz_5079f7eb
       field: _tmp.tz
       copy_from: _conf.tz_offset
       if: ctx._conf?.tz_offset != null && ctx._conf?.tz_offset != 'local'
   - set:
+      tag: set__tmp_tz_43d96fc2
       field: _tmp.tz
       copy_from: event.timezone
       override: false
       if: ctx.event?.timezone != null
   - set:
+      tag: set__tmp_tz_b8df3735
       field: _tmp.tz
       value: UTC
       override: false
   - set:
+      tag: set_event_timezone_ba6cb968
       field: event.timezone
       copy_from: _tmp.tz
   - date:
+      tag: date__tmp_timestamp_11c130e5
       field: _tmp.timestamp
       timezone: "{{{ event.timezone }}}"
       if: ctx._tmp?.timestamp != null
@@ -63,9 +75,11 @@ processors:
       on_failure:
         # Try to re-parse as UTC to catch when TZ is invalid or unknown.
         - remove:
+            tag: remove_event_timezone_aeeaa3d5
             field: event.timezone
             ignore_missing: true
         - date:
+            tag: date__tmp_timestamp_c043b8fc
             field: _tmp.timestamp
             if: ctx._tmp?.timestamp != null
             formats:
@@ -76,59 +90,76 @@ processors:
               - MMM dd HH:mm:ss
               - MMM d HH:mm:ss
   - pipeline:
+      tag: pipeline_e1b0fc56
       name: '{{ IngestPipeline "pipeline_authentication" }}'
       if: ctx.cisco_secure_email_gateway?.log?.category?.name == 'authentication'
   - pipeline:
+      tag: pipeline_ad806cb2
       name: '{{ IngestPipeline "pipeline_gui_logs" }}'
       if: ctx.cisco_secure_email_gateway?.log?.category?.name == 'gui_logs'
   - pipeline:
+      tag: pipeline_9b0b8b89
       name: '{{ IngestPipeline "pipeline_anti_spam" }}'
       if: ctx.cisco_secure_email_gateway?.log?.category?.name == 'antispam'
   - pipeline:
+      tag: pipeline_928f3e7a
       name: '{{ IngestPipeline "pipeline_error_logs" }}'
       if: ctx.cisco_secure_email_gateway?.log?.category?.name == 'error_logs'
   - pipeline:
+      tag: pipeline_5281e984
       name: '{{ IngestPipeline "pipeline_text_mail_logs" }}'
       if: ctx.cisco_secure_email_gateway?.log?.category?.name == 'mail_logs'
   - pipeline:
+      tag: pipeline_d6dcedc4
       name: '{{ IngestPipeline "pipeline_content_scanner" }}'
       if: ctx.cisco_secure_email_gateway?.log?.category?.name == 'content_scanner'
   - pipeline:
+      tag: pipeline_580b3b8e
       name: '{{ IngestPipeline "pipeline_system" }}'
       if: ctx.cisco_secure_email_gateway?.log?.category?.name == 'system'
   - pipeline:
+      tag: pipeline_1f40bd85
       name: '{{ IngestPipeline "pipeline_bounce" }}'
       if: ctx.cisco_secure_email_gateway?.log?.category?.name == 'bounces'
   - pipeline:
+      tag: pipeline_181277ba
       name: '{{ IngestPipeline "pipeline_status" }}'
       if: ctx.cisco_secure_email_gateway?.log?.category?.name == 'status'
   - pipeline:
+      tag: pipeline_34c6b064
       name: '{{ IngestPipeline "pipeline_amp" }}'
       if: ctx.cisco_secure_email_gateway?.log?.category?.name == 'amp'
   - pipeline:
+      tag: pipeline_7d301c2a
       name: '{{ IngestPipeline "pipeline_consolidated_event" }}'
       if: '["consolidated_event", "SplunkGIS"].contains(ctx.cisco_secure_email_gateway?.log?.category?.name)'
   - pipeline:
+      tag: pipeline_80b3dc7a
       name: '{{ IngestPipeline "pipeline_antivirus" }}'
       if: ctx.cisco_secure_email_gateway?.log?.category?.name == 'antivirus'
   - set:
+      tag: set_email_from_address_fff7c0a5
       field: email.from.address
       value: ['{{{email.from.address}}}']
       if: ctx.email?.from?.address instanceof String
   - set:
+      tag: set_email_to_address_52631338
       field: email.to.address
       value: ['{{{email.to.address}}}']
       if: ctx.email?.to?.address instanceof String
   - set:
+      tag: set_event_category_d7f508d2
       field: event.category
       value: vulnerability
       if: ctx.cisco_secure_email_gateway?.log?.category?.name == 'antivirus'
   - remove:
+      tag: remove_3f586893
       field:
         - _tmp
         - _conf
       ignore_missing: true
   - script:
+      tag: script_b274a7df
       lang: painless
       source: |
         boolean dropEmptyFields(Object object) {
@@ -150,5 +181,8 @@ on_failure:
       value: pipeline_error
   - append:
       field: error.message
-      value: |-
-        Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'

packages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/pipeline_amp.yml

@@ -1,9 +1,11 @@
 ---
 processors:
   - set:
+      tag: set_event_kind_de80643c
       field: event.kind
       value: event
   - grok:
+      tag: grok_cisco_secure_email_gateway_log_message_4f15232d
       field: cisco_secure_email_gateway.log.message
       patterns:
         - '^File reputation query initiating. %{GREEDYDATA:_tmp.new_message}$'
@@ -15,127 +17,156 @@ processors:
         - '^Retrospective verdict received. %{GREEDYDATA:_tmp.new_message}$'
         - '^%{GREEDYDATA:cisco_secure_email_gateway.log.message}$'
   - kv:
+      tag: kv__tmp_new_message_0bb63133
       field: _tmp.new_message
       if: ctx._tmp.new_message != null
       field_split: ",\\s*(?=[^',]*[=:][^',]*)"
       value_split: '\s*=\s*|:\s*'
   - grok:
+      tag: grok__tmp_cisco_secure_email_gateway_log_remaining_details_7273cf72
       field: _tmp.cisco_secure_email_gateway.log.remaining_details
       if: ctx._tmp?.cisco_secure_email_gateway?.log?.remaining_details != null
       patterns:
         - '^File SHA256\[%{GREEDYDATA:email.attachments.file.hash.sha256}\] file mime\[%{GREEDYDATA:email.attachments.file.mime_type}\], upload priority\[%{GREEDYDATA:cisco_secure_email_gateway.log.upload.priority}\] not uploaded, re-tries\[%{GREEDYDATA:cisco_secure_email_gateway.log.retries:long}\], backoff\[%{GREEDYDATA:cisco_secure_email_gateway.log.backoff:long}\] %{GREEDYDATA:cisco_secure_email_gateway.log.details}$'
   - rename:
+      tag: rename_Timestamp_to__tmp_submit_timestamp_1031a728
       field: Timestamp
       target_field: _tmp.submit.timestamp
       ignore_missing: true
   - date:
+      tag: date__tmp_submit_timestamp_to_cisco_secure_email_gateway_log_submit_timestamp_dd83046a
       field: _tmp.submit.timestamp
       target_field: cisco_secure_email_gateway.log.submit.timestamp
       if: ctx._tmp?.submit?.timestamp != null && ctx.cisco_secure_email_gateway?.log?._tmp?.submit?.timestamp != '0'
       formats:
         - UNIX
       on_failure:
         - append:
+            tag: append_error_message_c26fa851
             field: error.message
             value: '{{{_ingest.on_failure_message}}}'
   - date:
+      tag: date__tmp_update_timestamp_to_cisco_secure_email_gateway_log_update_timestamp_5d6d9d79
       field: _tmp.update.timestamp
       target_field: cisco_secure_email_gateway.log.update.timestamp
       if: ctx._tmp.update?.timestamp != null && ctx.cisco_secure_email_gateway?.log?._tmp?.update?.timestamp != '0'
       formats:
         - UNIX
       on_failure:
         - append:
+            tag: append_error_message_5fc2eb40
             field: error.message
             value: '{{{_ingest.on_failure_message}}}'
   - rename:
+      tag: rename_File_Name_to_email_attachments_file_name_b9cd094c
       field: File Name
       target_field: email.attachments.file.name
       ignore_missing: true
   - rename:
+      tag: rename_MID_to_email_message_id_a1506dc1
       field: MID
       target_field: email.message_id
       ignore_missing: true
   - gsub:
+      tag: gsub_File_Size_07980b8f
       field: 'File Size'
       pattern: '\ bytes'
       replacement: ''
       ignore_failure: true
   - convert:
+      tag: convert_File_Size_to_email_attachments_file_size_1e170071
       field: 'File Size'
       target_field: email.attachments.file.size
       type: long
       ignore_missing: true
       on_failure:
         - remove:
+            tag: remove_File_Size_d7be2622
             field: 'File Size'
         - append:
+            tag: append_error_message_c78edce4
             field: error.message
             value: '{{{_ingest.on_failure_message}}}'
   - rename:
+      tag: rename_File_Type_to_email_content_type_16c2f40c
       field: 'File Type'
       target_field: email.content_type
       ignore_missing: true
   - rename:
+      tag: rename_FileName_to_email_attachments_file_name_0ccba13e
       field: FileName
       target_field: email.attachments.file.name
       ignore_missing: true
   - rename:
+      tag: rename_Malware_to_cisco_secure_email_gateway_log_malware_c3847855
       field: Malware
       target_field: cisco_secure_email_gateway.log.malware
       ignore_missing: true
   - rename:
+      tag: rename_Disposition_to_cisco_secure_email_gateway_log_disposition_855f2689
       field: Disposition
       target_field: cisco_secure_email_gateway.log.disposition
       ignore_missing: true
   - convert:
+      tag: convert_Analysis_Score_cccaa177
       field: 'Analysis Score'
       type: long
       ignore_missing: true
   - rename:
+      tag: rename_Analysis_Score_to_cisco_secure_email_gateway_log_score_2aaa7d11
       field: 'Analysis Score'
       target_field: cisco_secure_email_gateway.log.score
       ignore_missing: true
   - rename:
+      tag: rename_sha256_to_email_attachments_file_hash_sha256_d9ce7f1c
       field: sha256
       target_field: email.attachments.file.hash.sha256
       ignore_missing: true
   - rename:
+      tag: rename_upload_action_to_cisco_secure_email_gateway_log_upload_action_08c9aeac
       field: upload_action
       target_field: cisco_secure_email_gateway.log.upload.action
       ignore_missing: true
   - rename:
+      tag: rename_Reputation_Score_to_cisco_secure_email_gateway_log_reputation_score_f7e1261a
       field: 'Reputation Score'
       target_field: cisco_secure_email_gateway.log.reputation_score
       ignore_missing: true
   - rename:
+      tag: rename_SHA256_to_email_attachments_file_hash_sha256_a100e0fc
       field: SHA256
       target_field: email.attachments.file.hash.sha256
       ignore_missing: true
   - rename:
+      tag: rename_Spyname_to_cisco_secure_email_gateway_log_spy_name_53fe4fbe
       field: Spyname
       target_field: cisco_secure_email_gateway.log.spy_name
       ignore_missing: true
   - rename:
+      tag: rename_Verdict_to_cisco_secure_email_gateway_log_verdict_094b3001
       field: Verdict
       target_field: cisco_secure_email_gateway.log.verdict
       ignore_missing: true
   - rename:
+      tag: rename_verdict_source_to_cisco_secure_email_gateway_log_verdict_source_869c1997
       field: verdict_source
       target_field: cisco_secure_email_gateway.log.verdict_source
       ignore_missing: true
   - gsub:
+      tag: gsub_email_attachments_file_name_e3527757
       field: email.attachments.file.name
       pattern: \'
       replacement: ''
       ignore_failure: true
   - append:
+      tag: append_related_hash_227eb64c
       field: related.hash
       value: '{{{email.attachments.file.hash.sha256}}}'
       if: ctx.email?.attachments?.file?.hash?.sha256 != null
       allow_duplicates: false
       ignore_failure: true
   - remove:
+      tag: remove_fe577437
       field:
         - _tmp
         - 'File Size'
@@ -146,4 +177,8 @@ on_failure:
       value: pipeline_error
   - append:
       field: error.message
-      value: '{{{ _ingest.on_failure_message }}}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'

packages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/pipeline_anti_spam.yml

@@ -1,9 +1,11 @@
 ---
 processors:
   - set:
+      tag: set_event_kind_de80643c
       field: event.kind
       value: event
   - grok:
+      tag: grok_cisco_secure_email_gateway_log_message_2188ef85
       field: cisco_secure_email_gateway.log.message
       patterns:
         - '^case %{GREEDYDATA:cisco_secure_email_gateway.log.object_category} \(%{NUMBER:cisco_secure_email_gateway.log.case_id}\) : case-daemon: all %{DATA:cisco_secure_email_gateway.log.object} killed, %{GREEDYDATA:cisco_secure_email_gateway.log.result}$'
@@ -16,4 +18,8 @@ on_failure:
       value: pipeline_error
   - append:
       field: error.message
-      value: '{{{ _ingest.on_failure_message }}}'
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'