darktrace: handle nested defeat conditions in Darktrace models (#15552)

Darktrace model defeats are not 1 level only (List) as currently
implemented. Latest API provides functionality for nested conditions.
If not nested, the defeatID is an integer and the defeats list is a
list of conditions. However, if nested, defeats list is a list of lists
of conditions and defeatIDs can either be integers (e.g. 1) or strings
(e.g. 1-1, 1-2 etc.).

Definition of the field from Darktrace documentation:

    defeats.defeatID: numeric/string

    An id for the defeat within the context of the model. If defeats are
    conditional, their id will indicate the relationship - e.g., a
    condition for defeat with id 1 will have ID 1-1. In this case, the
    id will be a string.

Author: arvchristos

packages/darktrace/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "2.0.0"
+  changes:
+    - description: Fix model defeat handling when nested conditions exist and so defeat ID can be non-numeric. 
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15552
+    - description: '`darktrace.model_breach_alert.model.defeats.id` field type is changed from `long` to `keyword`.'
+      type: breaking-change
+      link: https://github.com/elastic/integrations/pull/15552
 - version: "1.23.0"
   changes:
     - description: Update Kibana constraint to support 9.0.0.

packages/darktrace/data_stream/model_breach_alert/_dev/test/pipeline/test-model-breach-alert.log

@@ -2053,23 +2053,23 @@
                                 },
                                 "comparator": "matches",
                                 "filtertype": "Connection hostname",
-                                "id": 1
+                                "id": "1"
                             },
                             {
                                 "arguments": {
                                     "value": "some.connection"
                                 },
                                 "comparator": "matches",
                                 "filtertype": "Connection hostname",
-                                "id": 2
+                                "id": "2"
                             },
                             {
                                 "arguments": {
                                     "value": "another.connection"
                                 },
                                 "comparator": "matches",
                                 "filtertype": "Connection hostname",
-                                "id": 3
+                                "id": "3"
                             }
                         ],
                         "delay": 0,
@@ -2259,6 +2259,249 @@
                 "preserve_original_event",
                 "preserve_duplicate_custom_fields"
             ]
+        },
+        {
+          "@timestamp": "2024-12-10T06:49:49.000Z",
+          "ecs": {
+            "version": "8.11.0"
+          },
+          "related": {
+            "user": [
+              "System",
+              "[email protected]"
+            ]
+          },
+          "host": {
+            "type": "desktop",
+            "id": "18390"
+          },
+          "rule": {
+            "author": [
+              "System"
+            ],
+            "name": "Antigena::Network::Insider Threat::Antigena Internal Data Transfer Block",
+            "description": "A device has been blocked transferring large volumes of data internally.\n\nAction: Review the device's other alerts to see the activities the device has been conducting. Clear any active blocks if the alert was considered to be legitimate.",
+            "category": "Informational",
+            "uuid": "00000000-0000-0000-0000-000000000000",
+            "version": "17"
+          },
+          "event": {
+            "severity": 2,
+            "original": "{\"acknowledged\":{\"time\":1733838122000,\"username\":\"[email protected]\"},\"breachUrl\":\"https://redacted.local/#modelbreach/36725\",\"commentCount\":0,\"creationTime\":1733813400000,\"device\":{\"devicelabel\":\"test_device\",\"did\":18390,\"firstSeen\":1731062390000,\"generatedlabel\":true,\"lastSeen\":1736249138000,\"os\":\"Windows NT kernel\",\"sid\":-6,\"typelabel\":\"Desktop\",\"typename\":\"desktop\"},\"devicepercentscore\":44,\"devicescore\":0.439,\"did\":18390,\"model\":{\"actions\":{\"aianalyst\":{\"hypotheses\":[]},\"alert\":true,\"antigena\":{\"action\":\"automatic\",\"confirm\":false,\"duration\":3600,\"threshold\":\"20\"},\"breach\":true,\"model\":true,\"setPriority\":false,\"setTag\":false,\"setType\":false},\"active\":true,\"activeTimes\":{\"devices\":{},\"tags\":{},\"type\":\"exclusions\",\"version\":2},\"autoSuppress\":false,\"autoUpdatable\":true,\"autoUpdate\":true,\"behaviour\":\"decreasing\",\"category\":\"Informational\",\"compliance\":false,\"created\":{\"by\":\"System\"},\"defeats\":[[{\"arguments\":{\"value\":\"REDACTED_IP\"},\"comparator\":\"matches\",\"defeatID\":1,\"filtertype\":\"Destination IP\"},{\"arguments\":{\"value\":\"REDACTED_CIDR\"},\"comparator\":\"matches\",\"defeatID\":\"1-1\",\"filtertype\":\"Source IP\"},{\"arguments\":{\"value\":\"1150\"},\"comparator\":\"=\",\"defeatID\":\"1-2\",\"filtertype\":\"Destination port\"}]],\"delay\":0,\"description\":\"A device has been blocked transferring large volumes of data internally.\\n\\nAction: Review the device's other alerts to see the activities the device has been conducting. Clear any active blocks if the alert was considered to be legitimate.\",\"edited\":{\"by\":\"[email protected]\",\"userID\":17},\"interval\":3600,\"logic\":{\"data\":[{\"cid\":67318,\"weight\":1}],\"targetScore\":1,\"type\":\"weightedComponentList\",\"version\":1},\"mitre\":{\"tactics\":[\"defense-evasion\",\"exfiltration\"],\"techniques\":[\"T1030\",\"T1070.004\"]},\"modified\":\"2024-11-19 13:45:36\",\"name\":\"Antigena::Network::Insider Threat::Antigena Internal Data Transfer Block\",\"phid\":10256,\"pid\":90,\"priority\":2,\"sequenced\":false,\"sharedEndpoints\":false,\"tags\":[],\"throttle\":3600,\"uuid\":\"00000000-0000-0000-0000-000000000000\",\"version\":17},\"pbid\":36725,\"pbscore\":0.586,\"percentscore\":44,\"score\":0.439,\"time\":1733813389000,\"triggeredComponents\":[{\"cbid\":45710,\"chid\":77645,\"cid\":67318,\"interval\":3600,\"logic\":{\"data\":{\"left\":{\"left\":\"A\",\"operator\":\"AND\",\"right\":{\"left\":\"B\",\"operator\":\"AND\",\"right\":\"C\"}},\"operator\":\"OR\",\"right\":{\"left\":\"A\",\"operator\":\"AND\",\"right\":{\"left\":\"B\",\"operator\":\"AND\",\"right\":\"D\"}}},\"version\":\"v0.1\"},\"metric\":{\"label\":\"Model\",\"mlid\":234,\"name\":\"dtmodelbreach\"},\"size\":1,\"threshold\":0,\"time\":1733813388000,\"triggeredFilters\":[{\"arguments\":{\"value\":\"Unusual Activity / Internal Data Transfer\"},\"cfid\":340800,\"comparatorType\":\"matches\",\"filterType\":\"Message\",\"id\":\"A\",\"trigger\":{\"value\":\"Unusual Activity / Internal Data Transfer\"}},{\"arguments\":{\"value\":50},\"cfid\":340801,\"comparatorType\":\">\",\"filterType\":\"Strength of model breach\",\"id\":\"B\",\"trigger\":{\"value\":\"71\"}},{\"arguments\":{\"value\":16},\"cfid\":340802,\"comparatorType\":\"has tag\",\"filterType\":\"Tagged internal source\",\"id\":\"C\",\"trigger\":{\"tag\":{\"data\":{\"auto\":false,\"color\":92,\"description\":\"\",\"visibility\":\"Public\"},\"isReferenced\":true,\"name\":\"Antigena All\",\"restricted\":false,\"thid\":289,\"tid\":16},\"value\":\"16\"}},{\"arguments\":{},\"cfid\":340804,\"comparatorType\":\"display\",\"filterType\":\"Message\",\"id\":\"d1\",\"trigger\":{\"value\":\"Unusual Activity / Internal Data Transfer\"}}]}]}",
+            "risk_score": 0.439,
+            "risk_score_norm": 43.9,
+            "kind": "event",
+            "created": "2024-12-10T06:50:00.000Z",
+            "start": [
+              "2024-12-10T06:49:48.000Z"
+            ],
+            "action": "automatic",
+            "type": [
+              "info"
+            ],
+            "url": "https://redacted.local/#modelbreach/36725"
+          },
+          "darktrace": {
+            "model_breach_alert": {
+              "pb_score": 0.586,
+              "breach_url": {
+                "path": "/",
+                "fragment": "modelbreach/36725",
+                "original": "https://redacted.local/#modelbreach/36725",
+                "scheme": "https",
+                "domain": "redacted.local"
+              },
+              "is_acknowledged": false,
+              "acknowledged": {
+                "time": "2024-12-10T13:42:02.000Z",
+                "username": "[email protected]"
+              },
+              "creation_time": "2024-12-10T06:50:00.000Z",
+              "pbid": 36725,
+              "score": 0.439,
+              "time": "2024-12-10T06:49:49.000Z",
+              "comment": {
+                "count": 0
+              },
+              "model": {
+                "priority": 2,
+                "throttle": 3600,
+                "uuid": "00000000-0000-0000-0000-000000000000",
+                "version": 17,
+                "is_active": true,
+                "edited": {
+                  "by": "[email protected]"
+                },
+                "active_times": {
+                  "type": "exclusions",
+                  "version": 2
+                },
+                "phid": 10256,
+                "is_auto_update": true,
+                "pid": 90,
+                "is_shared_endpoints": false,
+                "delay": 0,
+                "description": "A device has been blocked transferring large volumes of data internally.\n\nAction: Review the device's other alerts to see the activities the device has been conducting. Clear any active blocks if the alert was considered to be legitimate.",
+                "defeats": [
+                  [
+                    {
+                      "id": "1",
+                      "comparator": "matches",
+                      "arguments": {
+                        "value": "REDACTED_IP"
+                      },
+                      "filtertype": "Destination IP"
+                    },
+                    {
+                      "id": "1-1",
+                      "comparator": "matches",
+                      "arguments": {
+                        "value": "REDACTED_CIDR"
+                      },
+                      "filtertype": "Source IP"
+                    },
+                    {
+                      "id": "1-2",
+                      "comparator": "=",
+                      "arguments": {
+                        "value": "1150"
+                      },
+                      "filtertype": "Destination port"
+                    }
+                  ]
+                ],
+                "is_auto_suppress": false,
+                "is_sequenced": false,
+                "in_compliance_behavior_category": false,
+                "is_auto_updatable": true,
+                "modified": "2024-11-19T13:45:36.000Z",
+                "name": "Antigena::Network::Insider Threat::Antigena Internal Data Transfer Block",
+                "behaviour": "decreasing",
+                "category": "Informational",
+                "created": {
+                  "by": "System"
+                },
+                "interval": 3600,
+                "logic": {
+                  "data_weighted_component_list": [
+                    {
+                      "weight": 1,
+                      "cid": 67318
+                    }
+                  ],
+                  "target_score": 1,
+                  "type": "weightedComponentList",
+                  "version": 1
+                },
+                "actions": {
+                  "is_alerting": true,
+                  "is_tag_set": false,
+                  "is_type_set": false,
+                  "antigena": {
+                    "action": "automatic",
+                    "duration": 3600,
+                    "threshold": 20,
+                    "is_confirm_by_human_operator": false
+                  },
+                  "model": true,
+                  "is_priority_set": false,
+                  "is_breach": true
+                }
+              },
+              "device_score": 0.439,
+              "device": {
+                "first_seen": "2024-11-08T10:39:50.000Z",
+                "type_label": "Desktop",
+                "type_name": "desktop",
+                "last_seen": "2025-01-07T11:25:38.000Z",
+                "sid": -6,
+                "did": 18390
+              },
+              "triggered_components": [
+                {
+                  "triggered_filters": [
+                    {
+                      "cfid": 340800,
+                      "comparator_type": "matches",
+                      "filter_type": "Message",
+                      "arguments": {
+                        "value": "Unusual Activity / Internal Data Transfer"
+                      },
+                      "id": "A",
+                      "trigger": {
+                        "value": "Unusual Activity / Internal Data Transfer"
+                      }
+                    },
+                    {
+                      "cfid": 340801,
+                      "comparator_type": ">",
+                      "filter_type": "Strength of model breach",
+                      "arguments": {
+                        "value": 50
+                      },
+                      "id": "B",
+                      "trigger": {
+                        "value": "71"
+                      }
+                    },
+                    {
+                      "cfid": 340802,
+                      "comparator_type": "has tag",
+                      "filter_type": "Tagged internal source",
+                      "arguments": {
+                        "value": 16
+                      },
+                      "id": "C",
+                      "trigger": {
+                        "value": "16",
+                        "tag": {
+                          "data": {
+                            "auto": false,
+                            "color": 92,
+                            "visibility": "Public"
+                          },
+                          "restricted": false,
+                          "name": "Antigena All",
+                          "thid": 289,
+                          "is_referenced": true,
+                          "tid": 16
+                        }
+                      }
+                    },
+                    {
+                      "cfid": 340804,
+                      "id": "d1",
+                      "trigger": {
+                        "value": "Unusual Activity / Internal Data Transfer"
+                      },
+                      "comparator_type": "display",
+                      "filter_type": "Message"
+                    }
+                  ],
+                  "cbid": 45710,
+                  "size": 1,
+                  "metric": {
+                    "name": "dtmodelbreach",
+                    "mlid": 234,
+                    "label": "Model"
+                  },
+                  "time": "2024-12-10T06:49:48.000Z",
+                  "interval": 3600,
+                  "threshold": 0,
+                  "logic": {
+                    "data": "{left={left=A, right={left=B, right=C, operator=AND}, operator=AND}, right={left=A, right={left=B, right=D, operator=AND}, operator=AND}, operator=OR}",
+                    "version": "v0.1"
+                  },
+                  "chid": 77645,
+                  "cid": 67318
+                }
+              ]
+            }
+          },
+          "tags": [
+                "preserve_original_event",
+                "preserve_duplicate_custom_fields"
+            ]
         }
     ]
 }

packages/darktrace/data_stream/model_breach_alert/_dev/test/pipeline/test-model-breach-alert.log-expected.json

@@ -669,14 +669,8 @@ processors:
         convert:
           field: _ingest._value.defeatID
           target_field: _ingest._value.id
-          type: long
+          type: string
           ignore_missing: true
-          on_failure:
-            - remove:
-                field: _ingest._value.defeatID
-            - append:
-                field: error.message
-                value: '{{{_ingest.on_failure_message}}}'
   - foreach:
       field: json.model.defeats
       if: ctx.json?.model?.defeats instanceof List
@@ -685,6 +679,39 @@ processors:
         remove:
           field: _ingest._value.defeatID
           ignore_missing: true
+  - foreach:
+      field: json.model.defeats
+      if: ctx.json?.model?.defeats instanceof List
+      ignore_failure: true
+      processor:
+        foreach:
+          field: _ingest._value
+          ignore_failure: true
+          processor:
+            convert:
+              field: _ingest._value.defeatID
+              target_field: _ingest._value.id
+              type: string
+              ignore_missing: true
+              on_failure:
+                - remove:
+                    field: _ingest._value.defeatID
+                - append:
+                    field: error.message
+                    value: '{{{_ingest.on_failure_message}}}'
+  - foreach:
+      field: json.model.defeats
+      if: ctx.json?.model?.defeats instanceof List
+      ignore_failure: true
+      processor:
+        foreach:
+          field: _ingest._value
+          ignore_failure: true
+          processor:
+            remove:
+              field: _ingest._value.defeatID
+              ignore_missing: true
+
   - rename:
       field: json.model.defeats
       target_field: darktrace.model_breach_alert.model.defeats

packages/darktrace/data_stream/model_breach_alert/elasticsearch/ingest_pipeline/default.yml

@@ -225,7 +225,7 @@
               type: keyword
               description: The filter the defeat is made from.
             - name: id
-              type: long
+              type: keyword
               description: A unique ID for the defeat.
         - name: delay
           type: long

packages/darktrace/data_stream/model_breach_alert/fields/fields.yml

@@ -1086,7 +1086,7 @@ An example event for `model_breach_alert` looks as following:
 | darktrace.model_breach_alert.model.defeats.arguments.value | The value(s) that must match for the defeat to take effect. | keyword |
 | darktrace.model_breach_alert.model.defeats.comparator | The comparator that the value is compared against the create the defeat. | keyword |
 | darktrace.model_breach_alert.model.defeats.filtertype | The filter the defeat is made from. | keyword |
-| darktrace.model_breach_alert.model.defeats.id | A unique ID for the defeat. | long |
+| darktrace.model_breach_alert.model.defeats.id | A unique ID for the defeat. | keyword |
 | darktrace.model_breach_alert.model.delay | Minimum delay in seconds after a positive-scoring component has fired before the overall model score is calculated. Only applicable in target score models. | long |
 | darktrace.model_breach_alert.model.description | The optional description of the model. | keyword |
 | darktrace.model_breach_alert.model.edited.by | Username that last edited the model. | keyword |

packages/darktrace/docs/README.md

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: darktrace
 title: Darktrace
-version: "1.23.0"
+version: "2.0.0"
 description: Collect logs from Darktrace with Elastic Agent.
 type: integration
 categories: