From 381a7b5fd4a72437e7b0fc03293065853a04160a Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Wed, 22 Oct 2025 07:53:40 -0500 Subject: [PATCH 1/2] [fortinet_fortigate] Generate processor tags and normalize error handler - Generate tags for processors missing tags - Normalize the pipeline error handler --- packages/fortinet_fortigate/changelog.yml | 5 + .../elasticsearch/ingest_pipeline/default.yml | 123 ++++++++++++++++-- .../elasticsearch/ingest_pipeline/event.yml | 61 ++++++++- .../elasticsearch/ingest_pipeline/login.yml | 29 ++++- .../elasticsearch/ingest_pipeline/traffic.yml | 61 ++++++++- .../log/elasticsearch/ingest_pipeline/utm.yml | 106 ++++++++++++++- packages/fortinet_fortigate/manifest.yml | 2 +- 7 files changed, 369 insertions(+), 18 deletions(-) diff --git a/packages/fortinet_fortigate/changelog.yml b/packages/fortinet_fortigate/changelog.yml index 8287a78f0d4..c677116205f 100644 --- a/packages/fortinet_fortigate/changelog.yml +++ b/packages/fortinet_fortigate/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.34.1" + changes: + - description: Generate processor tags and normalize error handler. + type: enhancement + link: https://github.com/elastic/integrations/pull/15540 - version: "1.34.0" changes: - description: Copy xauthuser to source.user.name for vpn logs, add mapping for advpnsc field. diff --git a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 94909a8256b..d6f81abd5d1 100644 --- a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -2,20 +2,25 @@ description: Pipeline for parsing fortinet firewall logs processors: - set: + tag: set_ecs_version_f5923549 field: ecs.version value: '8.17.0' - set: + tag: set_event_original_b9758751 field: event.original copy_from: message - remove: + tag: remove_message_7144efd2 field: message - grok: + tag: grok_event_original_995e7c66 field: event.original ecs_compatibility: v1 patterns: - "^(?:%{SYSLOG5424PRI}%{NONNEGINT} )+(?:%{TIMESTAMP_ISO8601}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII}) +(-|%{SYSLOG5424PRINTASCII}) +(-|%{SYSLOG5424PRINTASCII}) +(-|%{SYSLOG5424PRINTASCII}) +(?:%{GREEDYDATA:syslog5424_msg}|-|)" - "^(?:%{SYSLOG5424PRI} *)?%{GREEDYDATA:syslog5424_msg}$" - script: + tag: script_4a9d0c10 lang: painless source: | if (ctx.log?.syslog?.priority != null) { @@ -27,10 +32,12 @@ processors: ctx.log.syslog['facility'] = facility; } - gsub: + tag: gsub_syslog5424_msg_b526c35d field: syslog5424_msg pattern: "[\u0000-\u001F\u007F]" replacement: "" - script: + tag: script_2073fe5f lang: painless if: ctx.syslog5424_msg != null description: | @@ -97,6 +104,7 @@ processors: } ctx.fortinet.firewall = map; - script: + tag: script_dc02f280 lang: painless source: | def fw = ctx.fortinet?.firewall; @@ -106,45 +114,55 @@ processors: fw.entrySet().removeIf(entry -> entry.getValue() == "N/A" || pat.matcher(entry.getKey()).find()); } - set: + tag: set_observer_vendor_7e57c221 field: observer.vendor value: Fortinet - set: + tag: set_observer_product_19a6a908 field: observer.product value: Fortigate - set: + tag: set_observer_type_5dddf3ba field: observer.type value: firewall # Fortigate may set two timezones for an event. Use the first. - set: + tag: set_event_timezone_5c5e1e41 if: ctx.fortinet?.firewall?.tz != null && !(ctx.fortinet.firewall.tz instanceof List) field: event.timezone copy_from: fortinet.firewall.tz - set: + tag: set_event_timezone_1f1c0600 if: ctx.fortinet?.firewall?.tz instanceof List && ctx.fortinet.firewall.tz.length != 0 field: event.timezone copy_from: fortinet.firewall.tz.0 # The timezone may be formatted for human consumption, so fix that too. - gsub: + tag: gsub_event_timezone_1ce50a02 field: event.timezone pattern: ^(?:[A-Z]{1,4})([+-])([0-9]):?([0-9]{2})$ replacement: $10$2$3 ignore_missing: true ignore_failure: true - gsub: + tag: gsub_event_timezone_46d548b8 field: event.timezone pattern: ^(?:[A-Z]{1,4})([+-][0-9]{2}):?([0-9]{2})$ replacement: $1$2 ignore_missing: true ignore_failure: true - set: + tag: set__temp_time_2240d59c field: _temp.time value: "{{{fortinet.firewall.date}}} {{{fortinet.firewall.time}}} {{{event.timezone}}}" if: ctx.fortinet?.firewall?.date != null && ctx.fortinet?.firewall?.time != null && ctx.event?.timezone != null - set: + tag: set__temp_time_41a88693 field: _temp.time value: "{{{fortinet.firewall.date}}} {{{fortinet.firewall.time}}}" if: ctx.fortinet?.firewall?.date != null && ctx.fortinet?.firewall?.time != null && ctx.event?.timezone == null - date: + tag: date__temp_time_to_@timestamp_745b440f field: _temp.time target_field: "@timestamp" formats: @@ -155,6 +173,7 @@ processors: timezone: "{{{event.timezone}}}" if: ctx._temp?.time != null && ctx.event?.timezone != null - date: + tag: date__temp_time_to_@timestamp_978d46f3 field: _temp.time target_field: "@timestamp" formats: @@ -164,11 +183,13 @@ processors: - ISO8601 if: ctx._temp?.time != null && ctx.event?.timezone == null - gsub: + tag: gsub_fortinet_firewall_eventtime_7a46875e field: fortinet.firewall.eventtime pattern: "\\d{6}$" replacement: "" if: ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 18 - date: + tag: date_fortinet_firewall_eventtime_to_event_start_2c86beef field: fortinet.firewall.eventtime target_field: event.start formats: @@ -176,6 +197,7 @@ processors: timezone: "{{{event.timezone}}}" if: ctx.fortinet?.firewall?.eventtime != null && ctx.event?.timezone != null && (ctx.fortinet?.firewall?.eventtime).length() > 11 - date: + tag: date_fortinet_firewall_eventtime_to_event_start_5c1ae327 field: fortinet.firewall.eventtime target_field: event.start formats: @@ -183,97 +205,119 @@ processors: timezone: "{{{event.timezone}}}" if: ctx.fortinet?.firewall?.eventtime != null && ctx.event?.timezone != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11 - date: + tag: date_fortinet_firewall_eventtime_to_event_start_d17d3bd3 field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX_MS if: ctx.fortinet?.firewall?.eventtime != null && ctx.event?.timezone == null && (ctx.fortinet?.firewall?.eventtime).length() > 11 - date: + tag: date_fortinet_firewall_eventtime_to_event_start_c49d753b field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX if: ctx.fortinet?.firewall?.eventtime != null && ctx.event?.timezone == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11 - rename: + tag: rename_fortinet_firewall_devname_to_observer_name_81fc9d96 field: fortinet.firewall.devname target_field: observer.name ignore_missing: true - rename: + tag: rename_syslog5424_host_to_observer_name_193d2ffb field: syslog5424_host target_field: observer.name if: ctx.observer?.name == null && ctx.syslog5424_host !== null ignore_missing: true - remove: + tag: remove_c1748a95 field: - syslog5424_host - syslog5424_msg ignore_missing: true - script: + tag: script_587fe567 lang: painless source: "ctx.event.duration = Long.parseLong(ctx.fortinet.firewall.duration) * 1000000000" if: ctx.fortinet?.firewall?.duration != null - rename: + tag: rename_fortinet_firewall_devid_to_observer_serial_number_60129b13 field: fortinet.firewall.devid target_field: observer.serial_number ignore_missing: true - rename: + tag: rename_fortinet_firewall_dstintf_to_observer_egress_interface_name_340a1617 field: fortinet.firewall.dstintf target_field: observer.egress.interface.name ignore_missing: true if: ctx.observer?.egress?.interface?.name == null - rename: + tag: rename_fortinet_firewall_srcintf_to_observer_ingress_interface_name_2eb05d40 field: fortinet.firewall.srcintf target_field: observer.ingress.interface.name ignore_missing: true if: ctx.observer?.ingress?.interface?.name == null - rename: + tag: rename_fortinet_firewall_dst_int_to_observer_egress_interface_name_247e22bb field: fortinet.firewall.dst_int target_field: observer.egress.interface.name ignore_missing: true - rename: + tag: rename_fortinet_firewall_src_int_to_observer_ingress_interface_name_a72a5c7a field: fortinet.firewall.src_int target_field: observer.ingress.interface.name ignore_missing: true - rename: + tag: rename_fortinet_firewall_level_to_log_level_0e7843bb field: fortinet.firewall.level target_field: log.level ignore_missing: true - append: + tag: append_email_cc_address_4d235e7f field: email.cc.address value: "{{{fortinet.firewall.cc}}}" if: ctx.fortinet?.cc?.address != null - set: + tag: set_email_subject_5cfd5faa field: email.subject copy_from: fortinet.firewall.subject if: ctx.fortinet?.firewall?.subject != null - lowercase: + tag: lowercase_fortinet_firewall_srcname_to_host_name_bae14896 target_field: host.name field: fortinet.firewall.srcname if: ctx.fortinet?.firewall?.srcname != null - set: + tag: set_user_name_4fd1694d field: user.name copy_from: source.user.name if: ctx.source?.user?.name != null && ctx.user?.name == null - rename: + tag: rename_fortinet_firewall_msg_to_message_18251f47 field: fortinet.firewall.msg target_field: message ignore_missing: true - rename: + tag: rename_fortinet_firewall_dstmac_to_destination_mac_f4450c6e field: fortinet.firewall.dstmac target_field: destination.mac ignore_missing: true - gsub: + tag: gsub_destination_mac_52ddb47b field: destination.mac pattern: ':' replacement: '-' ignore_missing: true - uppercase: + tag: uppercase_destination_mac_04de3657 field: destination.mac ignore_missing: true - pipeline: + tag: pipeline_0b88bd91 name: '{{ IngestPipeline "event" }}' if: ctx.fortinet?.firewall?.type == 'event' - pipeline: + tag: pipeline_4025c313 name: '{{ IngestPipeline "login" }}' if: >- if (ctx.message instanceof String) { @@ -282,42 +326,51 @@ processors: } return false; - pipeline: + tag: pipeline_6d53bc4d name: '{{ IngestPipeline "traffic" }}' if: ctx.fortinet?.firewall?.type == 'traffic' - pipeline: + tag: pipeline_461401ce name: '{{ IngestPipeline "utm" }}' if: ctx.fortinet?.firewall?.type == 'utm' || ctx.fortinet?.firewall?.type == 'dns' - rename: + tag: rename_fortinet_firewall_eventtype_to_event_action_d051663b field: fortinet.firewall.eventtype target_field: event.action ignore_missing: true if: ctx.event?.action == null - rename: + tag: rename_fortinet_firewall_reason_to_event_reason_86432253 field: fortinet.firewall.reason target_field: event.reason ignore_missing: true if: ctx.event?.reason == null - rename: + tag: rename_fortinet_firewall_eventsubtype_to_event_reason_7e989cdd field: fortinet.firewall.eventsubtype target_field: event.reason ignore_missing: true if: ctx.event?.reason == null - rename: + tag: rename_fortinet_firewall_dir_to_network_direction_9bc88a5a field: fortinet.firewall.dir target_field: network.direction ignore_missing: true if: ctx.network?.direction == null - rename: + tag: rename_fortinet_firewall_direction_to_network_direction_a0716d1c field: fortinet.firewall.direction target_field: network.direction ignore_missing: true if: ctx.network?.direction == null - network_direction: + tag: network_direction_af8b6e3d internal_networks_field: _temp.internal_networks ignore_missing: true if: ctx.network?.direction == null # Handle interface-based network directionality - set: + tag: set_network_direction_0fbdc385 field: network.direction value: inbound if: > @@ -328,6 +381,7 @@ processors: ctx._temp.external_interfaces.contains(ctx.observer.ingress.interface.name) && ctx._temp.internal_interfaces.contains(ctx.observer.egress.interface.name) - set: + tag: set_network_direction_1f1faa3e field: network.direction value: outbound if: > @@ -338,6 +392,7 @@ processors: ctx._temp.external_interfaces.contains(ctx.observer.egress.interface.name) && ctx._temp.internal_interfaces.contains(ctx.observer.ingress.interface.name) - set: + tag: set_network_direction_fd72b4eb field: network.direction value: internal if: > @@ -348,6 +403,7 @@ processors: ctx._temp.internal_interfaces.contains(ctx.observer.egress.interface.name) && ctx._temp.internal_interfaces.contains(ctx.observer.ingress.interface.name) - set: + tag: set_network_direction_707f3fa1 field: network.direction value: external if: > @@ -358,6 +414,7 @@ processors: ctx._temp.external_interfaces.contains(ctx.observer.egress.interface.name) && ctx._temp.external_interfaces.contains(ctx.observer.ingress.interface.name) - set: + tag: set_network_direction_fdc5ffc3 field: network.direction value: unknown if: > @@ -377,6 +434,7 @@ processors: ) # Normalize the network direction - script: + tag: script_2ad0e621 lang: painless ignore_failure: true params: @@ -393,78 +451,96 @@ processors: if: ctx.network?.direction instanceof String # Fix up network direction field to match ECS-allowable values. - set: + tag: set_network_direction_8fd7edc5 field: network.direction value: unknown if: ctx.network?.direction != null && !(['ingress', 'egress', 'inbound', 'outbound', 'internal', 'external'].contains(ctx.network.direction)) - rename: + tag: rename_fortinet_firewall_interface_to_observer_ingress_interface_name_26e89051 field: fortinet.firewall.interface target_field: observer.ingress.interface.name ignore_missing: true if: ctx.observer?.ingress?.interface?.name == null && (['ingress', 'inbound', 'internal'].contains(ctx.network?.direction)) - rename: + tag: rename_fortinet_firewall_interface_to_observer_egress_interface_name_4a104736 field: fortinet.firewall.interface target_field: observer.egress.interface.name ignore_missing: true if: ctx.observer?.egress?.interface?.name == null && (['egress', 'outbound', 'external'].contains(ctx.network?.direction)) - convert: + tag: convert_fortinet_firewall_auditid_ec0373ea field: fortinet.firewall.auditid type: long ignore_missing: true - convert: + tag: convert_fortinet_firewall_audittime_0d65ad6e field: fortinet.firewall.audittime type: long ignore_missing: true - convert: + tag: convert_fortinet_firewall_quotamax_a2257e1c field: fortinet.firewall.quotamax type: long ignore_missing: true - convert: + tag: convert_fortinet_firewall_quotaused_6a99b037 field: fortinet.firewall.quotaused type: long ignore_missing: true - convert: + tag: convert_fortinet_firewall_size_4d138169 field: fortinet.firewall.size type: long ignore_missing: true - convert: + tag: convert_fortinet_firewall_disklograte_cc8fa1f9 field: fortinet.firewall.disklograte type: long ignore_missing: true - convert: + tag: convert_fortinet_firewall_fazlograte_0b3f0dbb field: fortinet.firewall.fazlograte type: long ignore_missing: true - convert: + tag: convert_fortinet_firewall_lanin_7938bde0 field: fortinet.firewall.lanin type: long ignore_missing: true - convert: + tag: convert_fortinet_firewall_lanout_826341f9 field: fortinet.firewall.lanout type: long ignore_missing: true - convert: + tag: convert_fortinet_firewall_setuprate_a6988fbd field: fortinet.firewall.setuprate type: long ignore_missing: true - convert: + tag: convert_fortinet_firewall_wanin_754b2213 field: fortinet.firewall.wanin type: long ignore_missing: true - convert: + tag: convert_fortinet_firewall_wanout_2bbe180c field: fortinet.firewall.wanout type: long ignore_missing: true - geoip: + tag: geoip_source_ip_to_source_geo_0e79e8a4 field: source.ip target_field: source.geo ignore_missing: true if: ctx.source?.geo == null - geoip: + tag: geoip_destination_ip_to_destination_geo_2f67bd6f field: destination.ip target_field: destination.geo ignore_missing: true if: ctx.destination?.geo == null - geoip: + tag: geoip_source_ip_to_source_as_28d69883 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -473,6 +549,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_to_destination_as_8a007787 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -481,16 +558,19 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_source_nat_ip_to_source_geo_9fb0fcc3 field: source.nat.ip target_field: source.geo ignore_missing: true if: ctx.source?.geo == null - geoip: + tag: geoip_destination_nat_ip_to_destination_geo_b429ceb4 field: destination.nat.ip target_field: destination.geo ignore_missing: true if: ctx.destination?.geo == null - geoip: + tag: geoip_source_nat_ip_to_source_as_d9d3c7d1 database_file: GeoLite2-ASN.mmdb field: source.nat.ip target_field: source.as @@ -500,6 +580,7 @@ processors: ignore_missing: true if: ctx.source?.as == null - geoip: + tag: geoip_destination_nat_ip_to_destination_as_dcccb906 database_file: GeoLite2-ASN.mmdb field: destination.nat.ip target_field: destination.as @@ -509,32 +590,39 @@ processors: ignore_missing: true if: ctx.destination?.as == null - rename: + tag: rename_source_as_asn_to_source_as_number_a917047d field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn_to_destination_as_number_3b459fcd field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name_to_destination_as_organization_name_814bd459 field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - script: + tag: script_7a41cd50 lang: painless source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes" if: ctx.source?.bytes != null && ctx.destination?.bytes != null ignore_failure: true - script: + tag: script_8baeb286 lang: painless source: "ctx.network.packets = ctx.source.packets + ctx.destination.packets" if: ctx.source?.packets != null && ctx.destination?.packets != null ignore_failure: true - script: + tag: script_c5ddceda lang: painless ignore_failure: true if: ctx.network?.iana_number != null @@ -564,90 +652,96 @@ processors: ctx.network.transport = 'sctp'; } - uppercase: + tag: uppercase_source_mac_5b4e7be2 field: source.mac ignore_missing: true - gsub: + tag: gsub_source_mac_3ceae5bc field: source.mac pattern: '[:.]' replacement: '-' ignore_missing: true - append: + tag: append_related_ip_8121c591 field: related.ip value: "{{{source.ip}}}" if: ctx.source?.ip != null allow_duplicates: false - append: + tag: append_related_ip_c1a6356b field: related.ip value: "{{{destination.ip}}}" if: ctx.destination?.ip != null allow_duplicates: false - append: - field: related.ip - value: "{{{source.nat.ip}}}" - if: ctx.source?.nat?.ip != null - allow_duplicates: false - - append: - field: related.ip - value: "{{{destination.nat.ip}}}" - if: ctx.destination?.nat?.ip != null - allow_duplicates: false - - append: + tag: append_related_ip_53b62ed8 field: related.ip value: "{{{source.nat.ip}}}" allow_duplicates: false if: ctx.source?.nat?.ip != null - append: + tag: append_related_ip_6a388074 field: related.ip value: "{{{destination.nat.ip}}}" - allow_duplicates: false if: ctx.destination?.nat?.ip != null + allow_duplicates: false - append: + tag: append_related_ip_1ccb183c field: related.ip value: "{{{fortinet.firewall.ip}}}" allow_duplicates: false if: ctx.fortinet?.firewall?.ip != null - append: + tag: append_related_ip_45cc2958 field: related.ip value: "{{{fortinet.firewall.assignip}}}" allow_duplicates: false if: ctx.fortinet?.firewall?.assignip != null - append: + tag: append_related_ip_a5b82964 field: related.ip value: "{{{fortinet.firewall.tunnelip}}}" allow_duplicates: false if: ctx.fortinet?.firewall?.tunnelip != null - append: + tag: append_related_user_f745a124 field: related.user value: "{{{source.user.name}}}" if: ctx.source?.user?.name != null allow_duplicates: false - append: + tag: append_related_user_8f37bc82 field: related.user value: "{{{destination.user.name}}}" if: ctx.destination?.user?.name != null allow_duplicates: false - append: + tag: append_related_hosts_aa05bed9 field: related.hosts value: "{{{destination.address}}}" if: ctx.destination?.address != null allow_duplicates: false - append: + tag: append_related_hosts_439b2767 field: related.hosts value: "{{{source.address}}}" if: ctx.source?.address != null allow_duplicates: false - append: + tag: append_related_hosts_452ef445 field: related.hosts value: "{{{host.name}}}" if: ctx.host?.name != null allow_duplicates: false - append: + tag: append_related_hosts_c4ad0d16 field: related.hosts value: "{{{dns.question.name}}}" if: ctx.dns?.question?.name != null allow_duplicates: false - script: + tag: script_b4f88753 lang: painless source: | def dnsIPs = ctx.dns?.resolved_ip; @@ -680,6 +774,7 @@ processors: } dropEmptyFields(ctx); - remove: + tag: remove_c87f67c3 field: - _temp - fortinet.firewall.tz @@ -695,4 +790,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/event.yml b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/event.yml index 9bec2c90512..24d9f9ccf9b 100644 --- a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/event.yml +++ b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/event.yml @@ -2,72 +2,88 @@ description: Pipeline for parsing fortinet firewall logs (event pipeline) processors: - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_outcome_102f0bb5 field: event.outcome value: failure if: ctx.fortinet?.firewall?.result == 'ERROR' || ctx.fortinet?.firewall?.status == 'negotiate_error' - set: + tag: set_event_outcome_bac8ed90 field: event.outcome value: success if: ctx.fortinet?.firewall?.result == 'OK' || ['FSSO-logon', 'auth-logon', 'FSSO-logoff', 'auth-logout'].contains(ctx.fortinet?.firewall?.action) - append: + tag: append_event_type_41eb50c6 field: event.type value: - start if: "['FSSO-logon', 'auth-logon'].contains(ctx.fortinet?.firewall?.action)" - append: + tag: append_event_type_a0b70a82 field: event.type value: - end if: "['FSSO-logoff', 'auth-logout'].contains(ctx.fortinet?.firewall?.action)" - append: + tag: append_event_type_7df30e03 field: event.type value: connection if: ctx.fortinet?.firewall?.subtype == 'vpn' - append: + tag: append_event_category_75e66583 field: event.category value: network if: ctx.fortinet?.firewall?.subtype == 'vpn' - append: + tag: append_event_type_d9e5777e field: event.type value: info if: ctx.fortinet?.firewall?.action == 'perf-stats' - append: + tag: append_event_category_c9e00984 field: event.category value: host if: ctx.fortinet?.firewall?.action == 'perf-stats' - append: + tag: append_event_type_0b9b5f72 field: event.type value: info if: ctx.fortinet?.firewall?.subtype == 'update' - append: + tag: append_event_category_61d5b2cf field: event.category value: - host - malware if: ctx.fortinet?.firewall?.subtype == 'update' - append: + tag: append_event_category_975470e4 field: event.category value: authentication if: ctx.fortinet?.firewall?.subtype == 'user' - rename: + tag: rename_fortinet_firewall_dstip_to_destination_ip_72bf25a6 field: fortinet.firewall.dstip target_field: destination.ip ignore_missing: true - rename: + tag: rename_fortinet_firewall_remip_to_destination_ip_971a6fc2 field: fortinet.firewall.remip target_field: destination.ip ignore_missing: true if: ctx.destination?.ip == null - convert: + tag: convert_fortinet_firewall_dstport_to_destination_port_37791676 field: fortinet.firewall.dstport target_field: destination.port type: long ignore_failure: true ignore_missing: true - convert: + tag: convert_fortinet_firewall_remport_to_destination_port_763b849e field: fortinet.firewall.remport target_field: destination.port type: long @@ -75,60 +91,72 @@ processors: ignore_missing: true if: ctx.destination?.port == null - convert: + tag: convert_fortinet_firewall_rcvdbyte_to_destination_bytes_65439a41 field: fortinet.firewall.rcvdbyte target_field: destination.bytes type: long ignore_failure: true ignore_missing: true - rename: + tag: rename_fortinet_firewall_daddr_to_destination_address_b6978056 field: fortinet.firewall.daddr target_field: destination.address ignore_missing: true - rename: + tag: rename_fortinet_firewall_dst_host_to_destination_address_8323a9e5 field: fortinet.firewall.dst_host target_field: destination.address ignore_missing: true if: ctx.destination?.address == null - rename: + tag: rename_fortinet_firewall_dst_host_to_destination_domain_f63213f3 field: fortinet.firewall.dst_host target_field: destination.domain ignore_missing: true if: ctx.destination?.address == null - rename: + tag: rename_fortinet_firewall_group_to_source_user_group_name_8ef934c4 field: fortinet.firewall.group target_field: source.user.group.name ignore_missing: true - convert: + tag: convert_fortinet_firewall_sentbyte_to_source_bytes_fe52ad03 field: fortinet.firewall.sentbyte target_field: source.bytes type: long ignore_failure: true ignore_missing: true - rename: + tag: rename_fortinet_firewall_srcip_to_source_ip_79344114 field: fortinet.firewall.srcip target_field: source.ip ignore_missing: true - rename: + tag: rename_fortinet_firewall_locip_to_source_ip_47ccd6f8 field: fortinet.firewall.locip target_field: source.ip ignore_missing: true if: ctx.source?.ip == null - rename: + tag: rename_fortinet_firewall_srcmac_to_source_mac_bfde8f82 field: fortinet.firewall.srcmac target_field: source.mac ignore_missing: true - rename: + tag: rename_fortinet_firewall_source_mac_to_source_mac_e364b602 field: fortinet.firewall.source_mac target_field: source.mac ignore_missing: true if: ctx.source?.mac == null - convert: + tag: convert_fortinet_firewall_srcport_to_source_port_dbc962e4 field: fortinet.firewall.srcport target_field: source.port type: long ignore_failure: true ignore_missing: true - convert: + tag: convert_fortinet_firewall_locport_to_source_port_ecd8b392 field: fortinet.firewall.locport target_field: source.port type: long @@ -136,72 +164,89 @@ processors: ignore_missing: true if: ctx.source?.port == null - rename: + tag: rename_fortinet_firewall_user_to_source_user_name_ba3b5509 field: fortinet.firewall.user target_field: source.user.name ignore_missing: true - rename: + tag: rename_fortinet_firewall_saddr_to_source_address_036ff596 field: fortinet.firewall.saddr target_field: source.address ignore_missing: true - rename: + tag: rename_fortinet_firewall_agent_to_user_agent_original_c612b9cc field: fortinet.firewall.agent target_field: user_agent.original ignore_missing: true - rename: + tag: rename_fortinet_firewall_file_to_file_name_a8d30898 field: fortinet.firewall.file target_field: file.name ignore_missing: true - convert: + tag: convert_fortinet_firewall_filesize_to_file_size_2e69dd69 field: fortinet.firewall.filesize target_field: file.size type: long ignore_failure: true ignore_missing: true - rename: + tag: rename_fortinet_firewall_level_to_log_level_0e7843bb field: fortinet.firewall.level target_field: log.level ignore_missing: true - rename: + tag: rename_fortinet_firewall_logid_to_event_code_d1ce9848 field: fortinet.firewall.logid target_field: event.code ignore_missing: true if: ctx.event?.code == null - rename: + tag: rename_fortinet_firewall_msg_to_message_18251f47 field: fortinet.firewall.msg target_field: message ignore_missing: true - rename: + tag: rename_fortinet_firewall_policyid_to_rule_id_b85da627 field: fortinet.firewall.policyid target_field: rule.id ignore_missing: true - rename: + tag: rename_fortinet_firewall_proto_to_network_iana_number_930125ae field: fortinet.firewall.proto target_field: network.iana_number ignore_missing: true - rename: + tag: rename_fortinet_firewall_service_to_network_protocol_c028ec88 field: fortinet.firewall.service target_field: network.protocol ignore_missing: true - lowercase: + tag: lowercase_network_protocol_49872259 field: network.protocol ignore_missing: true - rename: + tag: rename_fortinet_firewall_error_num_to_error_code_8e7c3151 field: fortinet.firewall.error_num target_field: error.code ignore_missing: true - rename: + tag: rename_fortinet_firewall_logdesc_to_rule_description_4783eec6 field: fortinet.firewall.logdesc target_field: rule.description ignore_missing: true - convert: + tag: convert_fortinet_firewall_addr_e65b2bfc field: fortinet.firewall.addr type: ip if: ctx.fortinet?.firewall?.addr != null && ctx.fortinet.firewall.addrgrp == null on_failure: - rename: + tag: rename_fortinet_firewall_addr_to_fortinet_firewall_addrgrp_5588fbc3 field: fortinet.firewall.addr target_field: fortinet.firewall.addrgrp - uri_parts: + tag: uri_parts_fortinet_firewall_url_to_url_3a7b2d2e field: fortinet.firewall.url target_field: url keep_original: false @@ -214,17 +259,21 @@ processors: # Need to do a set, then remove since rename w/ override # is not supported in 8.3.0 - set: + tag: set_url_domain_e4ace8d8 field: url.domain copy_from: fortinet.firewall.hostname ignore_empty_value: true override: true - remove: + tag: remove_fortinet_firewall_hostname_161ef624 field: fortinet.firewall.hostname ignore_missing: true - remove: + tag: remove_fortinet_firewall_url_5d47c5fc field: fortinet.firewall.url ignore_missing: true - convert: + tag: convert_fortinet_firewall_sess_duration_to_event_duration_022af8e6 field: fortinet.firewall.sess_duration type: long target_field: event.duration @@ -232,19 +281,23 @@ processors: ignore_missing: true if: ctx.event?.duration == null - convert: + tag: convert_fortinet_firewall_mem_4afb50de field: fortinet.firewall.mem type: integer ignore_failure: true ignore_missing: true - convert: + tag: convert_fortinet_firewall_jitter_6cb67bbc field: fortinet.firewall.jitter type: float ignore_missing: true - convert: + tag: convert_fortinet_firewall_latency_a02f5b8a field: fortinet.firewall.latency type: float ignore_missing: true - set: + tag: set_source_user_name_d0ef01fd field: source.user.name copy_from: fortinet.firewall.xauthuser if: ctx.fortinet?.firewall?.subtype == 'vpn' && ctx.fortinet.firewall.xauthuser != null @@ -256,6 +309,7 @@ processors: ctx.fortinet.firewall.advpnsc = ctx.fortinet.firewall.advpnsc != '0'; } - remove: + tag: remove_79b94f58 field: - fortinet.firewall.dstport - fortinet.firewall.remport @@ -269,6 +323,7 @@ processors: # For vpn subtype, remip(the client), is source, and locip(the firewall) is destination # https://docs.fortinet.com/document/fortigate/7.6.0/administration-guide/834425/understanding-vpn-related-logs - script: + tag: script_345a587b lang: painless if: ctx.fortinet?.firewall?.subtype == 'vpn' source: | @@ -285,4 +340,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/login.yml b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/login.yml index 89e700ebba0..8a8b1afe594 100644 --- a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/login.yml +++ b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/login.yml @@ -2,25 +2,31 @@ description: "Pipeline for parsing fortinet firewall logs for login events" processors: - set: + tag: set_event_kind_de80643c field: "event.kind" value: "event" - set: + tag: set_event_action_9396d084 field: "event.action" value: "login" override: false - append: + tag: append_event_category_c182a53a field: "event.category" value: - "authentication" - set: + tag: set_user_name_7ff007fb field: "user.name" value: "{{{ source.user.name }}}" if: ctx.source?.user?.name != null - append: + tag: append_user_roles_9f4ec36b field: "user.roles" value: "{{{ fortinet.firewall.adminprof }}}" if: ctx.fortinet?.firewall?.adminprof != null - append: + tag: append_source_user_roles_fa50adf0 field: "source.user.roles" value: "{{{ fortinet.firewall.adminprof }}}" if: ctx.fortinet?.firewall?.adminprof != null @@ -31,6 +37,7 @@ processors: if: "ctx.fortinet?.firewall?.userfrom != null && ctx.fortinet.firewall.userfrom.startsWith('JSON(')" on_failure: - append: + tag: append_error_message_ec5eda80 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' description: "JSON(192.168.0.10)" @@ -41,6 +48,7 @@ processors: tag: "event outcome" on_failure: - append: + tag: append_error_message_1be372ee field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' description: "User login/logout successful" @@ -51,6 +59,7 @@ processors: if: "ctx.message != null && ctx.message.startsWith('Login from ssh:')" on_failure: - append: + tag: append_error_message_3db62f63 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' description: "Login from ssh: Failed for philipp from 192.168.0.10 port 38654" @@ -61,6 +70,7 @@ processors: if: "ctx.message != null && ctx.message.startsWith('Administrator') && !ctx.message.toLowerCase().contains('logged in')" on_failure: - append: + tag: append_error_message_b6df5be7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' description: "Administrator philipp login failed from ssh(192.168.0.10) because of invalid ssh key" @@ -72,35 +82,43 @@ processors: if: "ctx.message != null && ctx.message.startsWith('Administrator') && ctx.message.toLowerCase().contains('logged in')" on_failure: - append: + tag: append_error_message_9b89cd96 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' description: "Administrator admin logged in successfully from jsconsole|ssh(172.16.200.254)" - set: + tag: set_event_outcome_4b7c0079 field: "event.outcome" value: "failure" if: "ctx.event?.outcome != null && ctx.event?.outcome.toLowerCase().startsWith('fail')" - set: + tag: set_event_outcome_bd391e8b field: "event.outcome" value: "success" if: "ctx.event?.outcome != null && ctx.event?.outcome.toLowerCase().startsWith('success')" - rename: + tag: rename_fortinet_firewall_log_id_to_event_id_14d08806 field: "fortinet.firewall.log_id" target_field: "event.id" ignore_missing: true - rename: + tag: rename_fortinet_firewall_pri_to_log_level_c2c890f0 field: "fortinet.firewall.pri" target_field: "log.level" ignore_missing: true - rename: + tag: rename_fortinet_firewall_device_id_to_observer_serial_number_f8ed8afb field: "fortinet.firewall.device_id" target_field: "observer.serial_number" ignore_missing: true - append: + tag: append_user_roles_217fc576 field: user.roles value: "{{{ _tmp.user.roles }}}" if: ctx._tmp?.user?.roles != null allow_duplicates: false - append: + tag: append_source_user_roles_a2c82efb field: source.user.roles value: "{{{ _tmp.user.roles }}}" if: ctx._tmp?.user?.roles != null @@ -112,6 +130,7 @@ processors: if: ctx.source?.port != null on_failure: - append: + tag: append_error_message_16f8f049 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -121,9 +140,11 @@ processors: if: ctx.fortinet?.firewall?.valid != null on_failure: - append: + tag: append_error_message_8d4e2e43 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: + tag: remove_723f70cc field: - "fortinet.firewall.adminprof" - "fortinet.firewall.userfrom" @@ -139,5 +160,9 @@ on_failure: field: "event.kind" value: "pipeline_error" - append: - field: "error.message" - value: "{{{ _ingest.on_failure_message }}}" \ No newline at end of file + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/traffic.yml b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/traffic.yml index 0d84c7fb82b..e6669954aa3 100644 --- a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/traffic.yml +++ b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/traffic.yml @@ -2,230 +2,283 @@ description: Pipeline for parsing fortinet firewall logs (traffic pipeline) processors: - set: + tag: set_event_kind_de80643c field: event.kind value: event - set: + tag: set_event_action_d2842801 field: event.action value: "{{{fortinet.firewall.action}}}" ignore_empty_value: true - set: + tag: set_event_outcome_e219d8f1 field: event.outcome value: success if: ctx.fortinet?.firewall?.action != null - append: + tag: append_event_category_7afdca3c field: event.category value: network - append: + tag: append_event_type_ab8d9d0e field: event.type value: connection - append: + tag: append_event_type_7a3e52b1 field: event.type value: start if: ctx.fortinet?.firewall?.action == 'start' - append: + tag: append_event_type_95005a3f field: event.type value: end if: ctx.fortinet?.firewall?.action != null && ctx.fortinet?.firewall?.action !='start' - append: + tag: append_event_type_a0c004ed field: event.type value: protocol if: ctx.fortinet?.firewall?.app != null && ctx.fortinet?.firewall?.action != 'deny' - append: + tag: append_event_type_b0751c98 field: event.type value: allowed if: ctx.fortinet?.firewall?.utmaction == null && ctx.fortinet?.firewall?.action != 'deny' - append: + tag: append_event_type_4099186f field: event.type value: denied if: ctx.fortinet?.firewall?.utmaction == 'block' || ctx.fortinet?.firewall?.action == 'deny' - rename: + tag: rename_fortinet_firewall_dstip_to_destination_ip_72bf25a6 field: fortinet.firewall.dstip target_field: destination.ip ignore_missing: true - convert: + tag: convert_fortinet_firewall_tranip_to_destination_nat_ip_7b6fb54b field: fortinet.firewall.tranip target_field: destination.nat.ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_fortinet_firewall_tranip_c5cd8948 field: fortinet.firewall.tranip - append: + tag: append_error_message_7bacc1e0 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_fortinet_firewall_dstport_to_destination_port_37791676 field: fortinet.firewall.dstport target_field: destination.port type: long ignore_failure: true ignore_missing: true - convert: + tag: convert_fortinet_firewall_tranport_to_destination_nat_port_0a1cdcdb field: fortinet.firewall.tranport target_field: destination.nat.port type: long ignore_failure: true ignore_missing: true - convert: + tag: convert_fortinet_firewall_rcvddelta_fbf14766 field: fortinet.firewall.rcvddelta type: long ignore_failure: true ignore_missing: true - convert: + tag: convert_fortinet_firewall_rcvdbyte_to_destination_bytes_65439a41 field: fortinet.firewall.rcvdbyte target_field: destination.bytes type: long ignore_failure: true ignore_missing: true - convert: + tag: convert_fortinet_firewall_rcvdpkt_to_destination_packets_523a1dfa field: fortinet.firewall.rcvdpkt target_field: destination.packets type: long ignore_failure: true ignore_missing: true - append: + tag: append_email_to_address_20a3bc20 field: email.to.address value: "{{{fortinet.firewall.dstcollectedemail}}}" if: ctx.fortinet?.firewall?.dstcollectedemail != null - rename: + tag: rename_fortinet_firewall_dstname_to_destination_address_55c832a1 field: fortinet.firewall.dstname target_field: destination.address ignore_missing: true - rename: + tag: rename_fortinet_firewall_dstunauthuser_to_destination_user_name_379463a0 field: fortinet.firewall.dstunauthuser target_field: destination.user.name ignore_missing: true - rename: + tag: rename_fortinet_firewall_group_to_source_user_group_name_8ef934c4 field: fortinet.firewall.group target_field: source.user.group.name ignore_missing: true - convert: + tag: convert_fortinet_firewall_sentdelta_5e362e81 field: fortinet.firewall.sentdelta type: long ignore_failure: true ignore_missing: true - convert: + tag: convert_fortinet_firewall_sentbyte_to_source_bytes_fe52ad03 field: fortinet.firewall.sentbyte target_field: source.bytes type: long ignore_failure: true ignore_missing: true - rename: + tag: rename_fortinet_firewall_srcdomain_to_source_domain_0f9ec89a field: fortinet.firewall.srcdomain target_field: source.domain ignore_missing: true - rename: + tag: rename_fortinet_firewall_srcip_to_source_ip_79344114 field: fortinet.firewall.srcip target_field: source.ip ignore_missing: true - rename: + tag: rename_fortinet_firewall_srcmac_to_source_mac_bfde8f82 field: fortinet.firewall.srcmac target_field: source.mac ignore_missing: true - convert: + tag: convert_fortinet_firewall_srcport_to_source_port_dbc962e4 field: fortinet.firewall.srcport target_field: source.port type: long ignore_failure: true ignore_missing: true - rename: + tag: rename_fortinet_firewall_unauthuser_to_source_user_name_45077c3a field: fortinet.firewall.unauthuser target_field: source.user.name ignore_missing: true - rename: + tag: rename_fortinet_firewall_user_to_source_user_name_5bbf0229 field: fortinet.firewall.user target_field: source.user.name ignore_missing: true if: ctx.source?.user?.name == null - append: + tag: append_email_from_address_bc766c69 field: email.from.address value: "{{{fortinet.firewall.collectedemail}}}" if: ctx.fortinet?.firewall?.collectedemail != null - convert: + tag: convert_fortinet_firewall_sentpkt_to_source_packets_ada35c44 field: fortinet.firewall.sentpkt target_field: source.packets type: long ignore_failure: true ignore_missing: true - convert: + tag: convert_fortinet_firewall_transip_to_source_nat_ip_c36fcafa field: fortinet.firewall.transip target_field: source.nat.ip type: ip ignore_missing: true on_failure: - remove: + tag: remove_fortinet_firewall_transip_91f00e56 field: fortinet.firewall.transip - append: + tag: append_error_message_421bc80b field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_fortinet_firewall_transport_to_source_nat_port_06b4f995 field: fortinet.firewall.transport target_field: source.nat.port type: long ignore_failure: true ignore_missing: true - rename: + tag: rename_fortinet_firewall_app_to_network_application_352bb492 field: fortinet.firewall.app target_field: network.application ignore_missing: true - rename: + tag: rename_fortinet_firewall_filename_to_file_name_d7a6e0d3 field: fortinet.firewall.filename target_field: file.name ignore_missing: true - rename: + tag: rename_fortinet_firewall_logid_to_event_code_d1ce9848 field: fortinet.firewall.logid target_field: event.code ignore_missing: true if: ctx.event?.code == null - rename: + tag: rename_fortinet_firewall_msg_to_message_18251f47 field: fortinet.firewall.msg target_field: message ignore_missing: true - rename: + tag: rename_fortinet_firewall_comment_to_rule_description_09652b08 field: fortinet.firewall.comment target_field: rule.description ignore_missing: true - rename: + tag: rename_fortinet_firewall_policyid_to_rule_id_220b73f8 field: fortinet.firewall.policyid target_field: rule.id ignore_missing: true if: ctx.rule?.id == null - rename: + tag: rename_fortinet_firewall_poluuid_to_rule_uuid_67e505b2 field: fortinet.firewall.poluuid target_field: rule.uuid ignore_missing: true - rename: + tag: rename_fortinet_firewall_policytype_to_rule_ruleset_1b162d39 field: fortinet.firewall.policytype target_field: rule.ruleset ignore_missing: true - rename: + tag: rename_fortinet_firewall_policyname_to_rule_name_d59c3713 field: fortinet.firewall.policyname target_field: rule.name ignore_missing: true - rename: + tag: rename_fortinet_firewall_appcat_to_rule_category_89544502 field: fortinet.firewall.appcat target_field: rule.category ignore_missing: true - gsub: + tag: gsub_rule_category_90e9a3c8 field: rule.category pattern: "\\." replacement: "-" ignore_missing: true - rename: + tag: rename_fortinet_firewall_proto_to_network_iana_number_930125ae field: fortinet.firewall.proto target_field: network.iana_number ignore_missing: true - rename: + tag: rename_fortinet_firewall_service_to_network_protocol_c028ec88 field: fortinet.firewall.service target_field: network.protocol ignore_missing: true - rename: + tag: rename_fortinet_firewall_srcthreatfeed_to_threat_feed_name_9d1d1b8e field: fortinet.firewall.srcthreatfeed target_field: threat.feed.name ignore_missing: true - lowercase: + tag: lowercase_network_protocol_49872259 field: network.protocol ignore_missing: true - uri_parts: + tag: uri_parts_fortinet_firewall_url_to_url_e619068b field: fortinet.firewall.url target_field: url keep_original: false @@ -241,9 +294,11 @@ processors: lang: painless source: ctx.fortinet.firewall.deltabytes = ctx.fortinet.firewall.rcvddelta + ctx.fortinet.firewall.sentdelta - remove: + tag: remove_fortinet_firewall_url_5d47c5fc field: fortinet.firewall.url ignore_missing: true - remove: + tag: remove_11027338 field: - fortinet.firewall.dstport - fortinet.firewall.tranport @@ -260,4 +315,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/utm.yml b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/utm.yml index e2fbd9fd840..2378d24a9ee 100644 --- a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/utm.yml +++ b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/utm.yml @@ -2,43 +2,53 @@ description: Pipeline for parsing fortinet firewall logs (utm pipeline) processors: - set: + tag: set_event_kind_de80643c field: event.kind value: event - append: + tag: append_event_type_b5f2b289 field: event.type value: denied if: "['block', 'blocked'].contains(ctx.fortinet?.firewall?.action)" - append: + tag: append_event_type_45c048ea field: event.type value: info if: ctx.fortinet?.firewall?.subtype == 'dns' - append: + tag: append_event_type_c8b37b6a field: event.type value: allowed if: "['pass', 'passthrough'].contains(ctx.fortinet?.firewall?.action)" - set: + tag: set_event_outcome_e219d8f1 field: event.outcome value: success if: ctx.fortinet?.firewall?.action != null - append: + tag: append_event_category_7afdca3c field: event.category value: network - rename: + tag: rename_fortinet_firewall_dstip_to_destination_ip_72bf25a6 field: fortinet.firewall.dstip target_field: destination.ip ignore_missing: true - rename: + tag: rename_fortinet_firewall_remip_to_destination_ip_971a6fc2 field: fortinet.firewall.remip target_field: destination.ip ignore_missing: true if: ctx.destination?.ip == null - convert: + tag: convert_fortinet_firewall_dst_port_to_destination_port_5c36a431 field: fortinet.firewall.dst_port target_field: destination.port type: long ignore_failure: true ignore_missing: true - convert: + tag: convert_fortinet_firewall_remport_to_destination_port_763b849e field: fortinet.firewall.remport target_field: destination.port type: long @@ -46,6 +56,7 @@ processors: ignore_missing: true if: ctx.destination?.port == null - convert: + tag: convert_fortinet_firewall_dstport_to_destination_port_1215c57d field: fortinet.firewall.dstport target_field: destination.port type: long @@ -53,34 +64,41 @@ processors: ignore_missing: true if: ctx.destination?.port == null - convert: + tag: convert_fortinet_firewall_rcvdbyte_to_destination_bytes_65439a41 field: fortinet.firewall.rcvdbyte target_field: destination.bytes type: long ignore_failure: true ignore_missing: true - rename: + tag: rename_fortinet_firewall_recipient_to_email_to_address_e1ed6c6b field: fortinet.firewall.recipient target_field: email.to.address ignore_missing: true - append: + tag: append_email_to_address_78c2982a field: email.to.address value: "{{{fortinet.firewall.recipient}}}" if: ctx.fortinet?.firewall?.recipient != null - rename: + tag: rename_fortinet_firewall_group_to_source_user_group_name_8ef934c4 field: fortinet.firewall.group target_field: source.user.group.name ignore_missing: true - rename: + tag: rename_fortinet_firewall_locip_to_source_ip_51dcb848 field: fortinet.firewall.locip target_field: source.ip ignore_missing: true - convert: + tag: convert_fortinet_firewall_locport_to_source_port_6bc2e2d4 field: fortinet.firewall.locport target_field: source.port type: long ignore_failure: true ignore_missing: true - convert: + tag: convert_fortinet_firewall_src_port_to_source_port_bf031661 field: fortinet.firewall.src_port target_field: source.port type: long @@ -88,6 +106,7 @@ processors: ignore_missing: true if: ctx.source?.port == null - convert: + tag: convert_fortinet_firewall_srcport_to_source_port_21df2082 field: fortinet.firewall.srcport target_field: source.port type: long @@ -95,192 +114,235 @@ processors: ignore_missing: true if: ctx.source?.port == null - convert: + tag: convert_fortinet_firewall_sentbyte_to_source_bytes_fe52ad03 field: fortinet.firewall.sentbyte target_field: source.bytes type: long ignore_failure: true ignore_missing: true - rename: + tag: rename_fortinet_firewall_srcdomain_to_source_domain_0f9ec89a field: fortinet.firewall.srcdomain target_field: source.domain ignore_missing: true - rename: + tag: rename_fortinet_firewall_srcip_to_source_ip_3df192ac field: fortinet.firewall.srcip target_field: source.ip ignore_missing: true if: ctx.source?.ip == null - rename: + tag: rename_fortinet_firewall_httpmethod_to_http_request_method_e47a5424 field: fortinet.firewall.httpmethod target_field: http.request.method ignore_missing: true - rename: + tag: rename_fortinet_firewall_referralurl_to_http_request_referrer_7efe6725 field: fortinet.firewall.referralurl target_field: http.request.referrer ignore_missing: true - rename: + tag: rename_fortinet_firewall_srcmac_to_source_mac_bfde8f82 field: fortinet.firewall.srcmac target_field: source.mac ignore_missing: true - rename: + tag: rename_fortinet_firewall_unauthuser_to_source_user_name_45077c3a field: fortinet.firewall.unauthuser target_field: source.user.name ignore_missing: true - rename: + tag: rename_fortinet_firewall_user_to_source_user_name_5bbf0229 field: fortinet.firewall.user target_field: source.user.name ignore_missing: true if: ctx.source?.user?.name == null - append: + tag: append_email_sender_address_37ff254c field: email.sender.address value: "{{{fortinet.firewall.sender}}}" if: ctx.fortinet?.firewall?.sender != null - append: + tag: append_email_from_address_868613c1 field: email.from.address value: "{{{fortinet.firewall.from}}}" if: ctx.fortinet?.firewall?.from != null - rename: + tag: rename_fortinet_firewall_agent_to_user_agent_original_c612b9cc field: fortinet.firewall.agent target_field: user_agent.original ignore_missing: true - rename: + tag: rename_fortinet_firewall_app_to_network_application_352bb492 field: fortinet.firewall.app target_field: network.application ignore_missing: true - rename: + tag: rename_fortinet_firewall_appcat_to_rule_category_89544502 field: fortinet.firewall.appcat target_field: rule.category ignore_missing: true - rename: + tag: rename_fortinet_firewall_applist_to_rule_ruleset_8752d1ee field: fortinet.firewall.applist target_field: rule.ruleset ignore_missing: true - rename: + tag: rename_fortinet_firewall_catdesc_to_rule_category_863824c0 field: fortinet.firewall.catdesc target_field: rule.category ignore_missing: true if: ctx.rule?.category == null - gsub: + tag: gsub_rule_category_ea9c0116 field: rule.category pattern: "\\." replacement: "-" ignore_missing: true if: ctx.rule?.category != null - rename: + tag: rename_fortinet_firewall_error_to_event_message_1eb40e8e field: fortinet.firewall.error target_field: event.message ignore_missing: true - rename: + tag: rename_fortinet_firewall_errorcode_to_event_code_0c462b81 field: fortinet.firewall.errorcode target_field: event.code ignore_missing: true - rename: + tag: rename_fortinet_firewall_event_id_to_event_id_ad65b8d8 field: fortinet.firewall.event_id target_field: event.id ignore_missing: true - rename: + tag: rename_fortinet_firewall_eventid_to_event_id_91348166 field: fortinet.firewall.eventid target_field: event.id ignore_missing: true if: ctx.event?.id == null - rename: + tag: rename_fortinet_firewall_filename_to_file_name_d7a6e0d3 field: fortinet.firewall.filename target_field: file.name ignore_missing: true - convert: + tag: convert_fortinet_firewall_filesize_to_file_size_2e69dd69 field: fortinet.firewall.filesize target_field: file.size type: long ignore_failure: true ignore_missing: true - rename: + tag: rename_fortinet_firewall_filetype_to_file_extension_0d6978f4 field: fortinet.firewall.filetype target_field: file.extension ignore_missing: true - rename: + tag: rename_fortinet_firewall_infectedfilename_to_file_name_470a39d2 field: fortinet.firewall.infectedfilename target_field: file.name ignore_missing: true if: ctx.file?.name == null - rename: + tag: rename_fortinet_firewall_infectedfilesize_to_file_size_6190f68c field: fortinet.firewall.infectedfilesize target_field: file.size ignore_missing: true if: ctx.file?.size == null - rename: + tag: rename_fortinet_firewall_infectedfiletype_to_file_extension_c1ee548f field: fortinet.firewall.infectedfiletype target_field: file.extension ignore_missing: true if: ctx.file?.extension == null - rename: + tag: rename_fortinet_firewall_matchedfilename_to_file_name_f9cea46a field: fortinet.firewall.matchedfilename target_field: file.name ignore_missing: true if: ctx.file?.name == null - rename: + tag: rename_fortinet_firewall_matchedfiletype_to_file_extension_56d8df57 field: fortinet.firewall.matchedfiletype target_field: file.extension ignore_missing: true if: ctx.file?.extension == null - rename: + tag: rename_fortinet_firewall_ipaddr_to_dns_resolved_ip_3e7ee47e field: fortinet.firewall.ipaddr target_field: dns.resolved_ip ignore_missing: true - split: + tag: split_dns_resolved_ip_61a17abc field: dns.resolved_ip separator: ", " ignore_missing: true - rename: + tag: rename_fortinet_firewall_level_to_log_level_0e7843bb field: fortinet.firewall.level target_field: log.level ignore_missing: true - rename: + tag: rename_fortinet_firewall_logid_to_event_code_d1ce9848 field: fortinet.firewall.logid target_field: event.code ignore_missing: true if: ctx.event?.code == null - rename: + tag: rename_fortinet_firewall_msg_to_message_18251f47 field: fortinet.firewall.msg target_field: message ignore_missing: true - rename: + tag: rename_fortinet_firewall_policy_id_to_rule_id_4b286d2d field: fortinet.firewall.policy_id target_field: rule.id ignore_missing: true if: ctx.rule?.id == null - rename: + tag: rename_fortinet_firewall_policyid_to_rule_id_220b73f8 field: fortinet.firewall.policyid target_field: rule.id ignore_missing: true if: ctx.rule?.id == null - rename: + tag: rename_fortinet_firewall_profile_to_rule_ruleset_0715e7bc field: fortinet.firewall.profile target_field: rule.ruleset ignore_missing: true if: ctx.rule?.ruleset == null - rename: + tag: rename_fortinet_firewall_proto_to_network_iana_number_930125ae field: fortinet.firewall.proto target_field: network.iana_number ignore_missing: true - rename: + tag: rename_fortinet_firewall_qclass_to_dns_question_class_dd343cd1 field: fortinet.firewall.qclass target_field: dns.question.class ignore_missing: true - rename: + tag: rename_fortinet_firewall_qname_to_dns_question_name_ada7e477 field: fortinet.firewall.qname target_field: dns.question.name ignore_missing: true - rename: + tag: rename_fortinet_firewall_qtype_to_dns_question_type_3bfd2a8d field: fortinet.firewall.qtype target_field: dns.question.type ignore_missing: true - rename: + tag: rename_fortinet_firewall_service_to_network_protocol_c028ec88 field: fortinet.firewall.service target_field: network.protocol ignore_missing: true - lowercase: + tag: lowercase_network_protocol_49872259 field: network.protocol ignore_missing: true - uri_parts: + tag: uri_parts_fortinet_firewall_url_to_url_bdb5970a field: fortinet.firewall.url target_field: url keep_original: false @@ -293,136 +355,167 @@ processors: # Need to do a set, then remove since rename w/ override # is not supported in 8.3.0 - set: + tag: set_url_domain_e4ace8d8 field: url.domain copy_from: fortinet.firewall.hostname ignore_empty_value: true override: true - remove: + tag: remove_fortinet_firewall_hostname_161ef624 field: fortinet.firewall.hostname ignore_missing: true - remove: + tag: remove_fortinet_firewall_url_5d47c5fc field: fortinet.firewall.url ignore_missing: true - rename: + tag: rename_fortinet_firewall_xid_to_dns_id_7c6ef4f8 field: fortinet.firewall.xid target_field: dns.id ignore_missing: true - append: + tag: append_tls_server_x509_subject_common_name_822f6daa field: tls.server.x509.subject.common_name value: "{{{fortinet.firewall.scertcname}}}" if: ctx.fortinet?.firewall?.scertcname != null - rename: + tag: rename_fortinet_firewall_scertissuer_to_tls_server_issuer_65a8bad6 field: fortinet.firewall.scertissuer target_field: tls.server.issuer ignore_missing: true - append: + tag: append_tls_server_x509_issuer_common_name_e1391a8d field: tls.server.x509.issuer.common_name value: "{{{tls.server.issuer}}}" if: ctx.tls?.server?.issuer != null - rename: + tag: rename_fortinet_firewall_ccertissuer_to_tls_client_issuer_5d819a7a field: fortinet.firewall.ccertissuer target_field: tls.client.issuer ignore_missing: true - append: + tag: append_tls_client_x509_issuer_common_name_3908abd9 field: tls.client.x509.issuer.common_name value: "{{{tls.client.issuer}}}" if: ctx.tls?.client?.issuer != null - rename: + tag: rename_fortinet_firewall_sender_to_tls_server_issuer_53552963 field: fortinet.firewall.sender target_field: tls.server.issuer ignore_missing: true - rename: + tag: rename_fortinet_firewall_issuer_to_tls_server_issuer_81110cbf field: fortinet.firewall.issuer target_field: tls.server.issuer ignore_missing: true if: ctx.tls?.server?.issuer == null - rename: + tag: rename_fortinet_firewall_authalgo_to_tls_server_x509_public_key_algorithm_a19bd0ed field: fortinet.firewall.authalgo target_field: tls.server.x509.public_key_algorithm ignore_missing: true - rename: + tag: rename_fortinet_firewall_keyalgo_to_tls_server_x509_public_key_algorithm_b7b7c24d field: fortinet.firewall.keyalgo target_field: tls.server.x509.public_key_algorithm ignore_missing: true if: ctx.tls?.server?.x509?.public_key_algorithm == null - rename: + tag: rename_fortinet_firewall_notbefore_to_tls_server_not_before_e72d5bec field: fortinet.firewall.notbefore target_field: tls.server.not_before ignore_missing: true if: ctx.tls?.server?.not_before == null - rename: + tag: rename_fortinet_firewall_notafter_to_tls_server_not_after_243506eb field: fortinet.firewall.notafter target_field: tls.server.not_after ignore_missing: true if: ctx.tls?.server?.not_after == null - rename: + tag: rename_fortinet_firewall_keysize_to_tls_server_x509_public_key_size_eb70b1c1 field: fortinet.firewall.keysize target_field: tls.server.x509.public_key_size ignore_missing: true if: ctx.tls?.server?.x509?.public_key_size == null - convert: + tag: convert_tls_server_x509_public_key_size_6513b5d4 field: tls.server.x509.public_key_size type: long ignore_missing: true - rename: + tag: rename_fortinet_firewall_sn_to_tls_server_x509_serial_number_74ef21d8 field: fortinet.firewall.sn target_field: tls.server.x509.serial_number ignore_missing: true if: ctx.tls?.server?.x509?.serial_number == null - rename: + tag: rename_fortinet_firewall_certhash_to_tls_server_hash_sha1_9490fc6f field: fortinet.firewall.certhash target_field: tls.server.hash.sha1 ignore_missing: true if: ctx.tls?.server?.hash?.sha1 == null - append: + tag: append_related_hash_6b51007e field: related.hash value: "{{{tls.server.hash.sha1}}}" allow_duplicates: false if: ctx.tls?.server?.hash?.sha1 != null - set: + tag: set_tls_server_x509_not_after_99449ca7 field: tls.server.x509.not_after copy_from: tls.server.not_after ignore_empty_value: true - set: + tag: set_tls_server_x509_not_before_639ff489 field: tls.server.x509.not_before copy_from: tls.server.not_before ignore_empty_value: true - split: + tag: split_fortinet_firewall_san_to_tls_server_x509_alternative_names_33c817cd field: fortinet.firewall.san separator: ";" target_field: tls.server.x509.alternative_names ignore_missing: true - append: + tag: append_tls_server_x509_alternative_names_cb12c914 field: tls.server.x509.alternative_names value: "{{{fortinet.firewall.cn}}}" allow_duplicates: false if: ctx.fortinet?.firewall?.cn != null - set: + tag: set_tls_client_x509_public_key_algorithm_c8e4b429 field: tls.client.x509.public_key_algorithm copy_from: tls.server.x509.public_key_algorithm ignore_empty_value: true - rename: + tag: rename_fortinet_firewall_kxcurve_to_tls_curve_4ba43a3d field: fortinet.firewall.kxcurve target_field: tls.curve ignore_missing: true - rename: + tag: rename_fortinet_firewall_cipher_to_tls_cipher_2440d05e field: fortinet.firewall.cipher target_field: tls.cipher ignore_missing: true - rename: + tag: rename_fortinet_firewall_sni_to_tls_client_server_name_640d85d0 field: fortinet.firewall.sni target_field: tls.client.server_name ignore_missing: true - set: + tag: set_destination_domain_1ee4539a field: destination.domain copy_from: tls.client.server_name ignore_empty_value: true if: ctx.destination?.domain == null - set: + tag: set_tls_established_6ae896b6 field: tls.established value: true if: ctx.fortinet?.firewall?.handshake == "full" - script: + tag: script_55eb70c0 lang: painless if: ctx.fortinet?.firewall?.tlsver instanceof String source: >- @@ -441,32 +534,39 @@ processors: ctx.tls.version += ".0"; } - append: + tag: append_vulnerability_category_e516aea2 field: vulnerability.category value: "{{{fortinet.firewall.dtype}}}" allow_duplicates: false if: ctx.fortinet?.firewall?.dtype instanceof String - rename: + tag: rename_fortinet_firewall_ref_to_event_reference_ac3e7de7 field: fortinet.firewall.ref target_field: event.reference ignore_missing: true - rename: + tag: rename_fortinet_firewall_filehash_to_fortinet_file_hash_crc32_b8cdcb97 field: fortinet.firewall.filehash target_field: fortinet.file.hash.crc32 ignore_missing: true - append: + tag: append_related_hash_af3ee53d field: related.hash value: "{{{fortinet.file.hash.crc32}}}" if: ctx.fortinet?.file?.hash?.crc32 != null # Populate ECS dns - registered_domain: + tag: registered_domain_dns_question_name_to_dns_question_24868ff9 field: dns.question.name target_field: dns.question ignore_missing: true ignore_failure: true - remove: + tag: remove_dns_question_domain_3403bb7d field: dns.question.domain ignore_missing: true - remove: + tag: remove_54bd5916 field: - fortinet.firewall.cn - fortinet.firewall.san @@ -489,4 +589,8 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/fortinet_fortigate/manifest.yml b/packages/fortinet_fortigate/manifest.yml index de01e13783c..ad1282541a2 100644 --- a/packages/fortinet_fortigate/manifest.yml +++ b/packages/fortinet_fortigate/manifest.yml @@ -1,6 +1,6 @@ name: fortinet_fortigate title: Fortinet FortiGate Firewall Logs -version: "1.34.0" +version: "1.34.1" description: Collect logs from Fortinet FortiGate firewalls with Elastic Agent. type: integration format_version: "3.0.3" From 7cdd55d5355d8c0437a5c8d9a0fb5e0053c0f6a6 Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Tue, 28 Oct 2025 07:43:03 -0500 Subject: [PATCH 2/2] cleanup tags --- .../data_stream/log/elasticsearch/ingest_pipeline/default.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/default.yml index d6f81abd5d1..c1d877a273d 100644 --- a/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/fortinet_fortigate/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -162,7 +162,7 @@ processors: value: "{{{fortinet.firewall.date}}} {{{fortinet.firewall.time}}}" if: ctx.fortinet?.firewall?.date != null && ctx.fortinet?.firewall?.time != null && ctx.event?.timezone == null - date: - tag: date__temp_time_to_@timestamp_745b440f + tag: date__temp_time_to_timestamp_745b440f field: _temp.time target_field: "@timestamp" formats: @@ -173,7 +173,7 @@ processors: timezone: "{{{event.timezone}}}" if: ctx._temp?.time != null && ctx.event?.timezone != null - date: - tag: date__temp_time_to_@timestamp_978d46f3 + tag: date__temp_time_to_timestamp_978d46f3 field: _temp.time target_field: "@timestamp" formats: