[WithSecure] Added the WithSecure Elements integration for collecting security incident and events

packages/withsecure_elements/changelog.yml

@@ -0,0 +1,5 @@
+- version: "0.1.0"
+  changes:
+    - description: "Initial release."
+      type: enhancement
+      link: "https://github.com/elastic/integrations/pull/15442"

packages/withsecure_elements/data_stream/incident_detections/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,6 @@
+---
+# Optional: Common configuration applied to all test files
+# This file is not required if tests contain all necessary data
+fields:
+  "@timestamp": "2023-08-09T12:10:43.537Z"
+

packages/withsecure_elements/data_stream/incident_detections/_dev/test/pipeline/test-detection.json

@@ -0,0 +1,7 @@
+{
+  "events": [
+    {
+      "message": "{\"id\":\"detection-12345\",\"incidentId\":\"2c902c73-e2a6-40fd-9532-257ee102e1c1\",\"detectionType\":\"malware\",\"severity\":\"high\",\"status\":\"active\",\"createdTimestamp\":\"2023-08-09T12:10:43.537Z\",\"updatedTimestamp\":\"2023-08-09T12:15:43.537Z\",\"source\":\"endpoint\",\"category\":\"MALWARE\",\"description\":\"Malware detected on endpoint\",\"device\":{\"id\":\"device-123\",\"name\":\"WORKSTATION-01\",\"ipAddress\":\"192.168.1.100\",\"os\":\"Windows 10\"},\"user\":{\"id\":\"user-123\",\"name\":\"John Doe\",\"email\":\"[email protected]\"},\"file\":{\"name\":\"malware.exe\",\"path\":\"C:\\\\Users\\\\John\\\\Downloads\\\\malware.exe\",\"hash\":\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\",\"size\":1024000},\"network\":{\"sourceIp\":\"192.168.1.100\",\"destinationIp\":\"10.0.0.1\",\"port\":443,\"protocol\":\"HTTPS\"},\"details\":{\"threatName\":\"Trojan.Generic.123456\",\"confidence\":95,\"behavior\":\"suspicious_network_activity\"}}"
+    }
+  ]
+}

packages/withsecure_elements/data_stream/incident_detections/_dev/test/pipeline/test-detection.json-expected.json

@@ -0,0 +1,37 @@
+{
+  "expected": [
+    {
+      "event": {
+        "dataset": "withsecure_elements.incident_detections",
+        "module": "withsecure_elements",
+        "category": ["threat"],
+        "type": ["info"],
+        "kind": "alert",
+        "provider": "withsecure_elements",
+        "action": "detected",
+        "outcome": "success",
+        "id": "detection-12345",
+        "created": "2023-08-09T12:10:43.537Z",
+        "start": "2023-08-09T12:10:43.537Z",
+        "end": "2023-08-09T12:15:43.537Z",
+        "severity": "high"
+      },
+      "withsecure": {
+        "incident": {
+          "detection": {
+            "id": "detection-12345",
+            "incident_id": "2c902c73-e2a6-40fd-9532-257ee102e1c1",
+            "detection_type": "malware",
+            "severity": "high",
+            "status": "active",
+            "created_timestamp": "2023-08-09T12:10:43.537Z",
+            "updated_timestamp": "2023-08-09T12:15:43.537Z",
+            "source": "endpoint",
+            "category": "MALWARE",
+            "description": "Malware detected on endpoint"
+          }
+        }
+      }
+    }
+  ]
+}

packages/withsecure_elements/data_stream/incident_detections/agent/stream/httpjson.yml.hbs

@@ -0,0 +1,61 @@
+config_version: 2
+interval: {{interval}}
+{{#if enable_request_tracer}}
+request.tracer.filename: "../../logs/httpjson/http-request-trace-*.ndjson"
+request.tracer.maxbackups: 5
+{{/if}}
+request.url: {{url}}/incidents/v1/incidents/{{incident_id}}/detections
+request.method: GET
+{{#if http_client_timeout}}
+request.timeout: {{http_client_timeout}}
+{{/if}}
+{{#if proxy_url}}
+request.proxy_url: {{proxy_url}}
+{{/if}}
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+auth.oauth2.client.id: {{client_id}}
+auth.oauth2.client.secret: {{client_secret}}
+auth.oauth2.token_url: {{url}}/as/token.oauth2
+auth.oauth2.endpoint_params:
+  grant_type: client_credentials
+  scope: connect.api.read
+request.transforms:
+  - set:
+      target: url.params.limit
+      value: '50'
+  - set:
+      target: url.params.order
+      value: 'desc'
+  - set:
+      target: header.User-Agent
+      value: "Elastic-WithSecure-Elements-Connector/1.0.1"
+  - set:
+      target: header.Accept
+      value: "application/json"
+response.split:
+  target: body.items
+  ignore_empty_value: true
+response.pagination:
+  - set:
+      target: url.params.anchor
+      value: '[[.last_response.body.nextAnchor]]'
+      fail_on_template_error: true
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#if preserve_duplicate_custom_fields}}
+  - preserve_duplicate_custom_fields
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+  {{processors}}
+{{/if}}

packages/withsecure_elements/data_stream/incident_detections/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,185 @@
+---
+processors:
+  # Decode JSON from message or event.original
+  - set:
+      field: event.original
+      copy_from: message
+      if: ctx.event?.original == null && ctx.message != null
+      ignore_failure: true
+  - remove:
+      field: message
+      if: ctx.event?.original != null && ctx.message != null && ctx.message == ctx.event.original
+      ignore_missing: true
+  - json:
+      field: event.original
+      target_field: _temp
+      if: ctx.event?.original != null && ctx.event.original instanceof String
+      ignore_failure: true
+      tag: json_decode
+      description: Decode JSON from event.original
+  - script:
+      lang: painless
+      description: Move JSON fields from _temp to root context
+      tag: move_json_fields
+      source: |
+        if (ctx._temp != null) {
+          for (entry in ctx._temp.entrySet()) {
+            ctx[entry.getKey()] = entry.getValue();
+          }
+          ctx.remove('_temp');
+        }
+      if: ctx._temp != null
+
+  # Set ECS fields
+  - set:
+      field: event.dataset
+      value: withsecure_elements.incident_detections
+  - set:
+      field: event.module
+      value: withsecure_elements
+  - set:
+      field: event.category
+      value: [threat]
+  - set:
+      field: event.type
+      value: [info]
+  - set:
+      field: event.kind
+      value: alert
+  - set:
+      field: event.provider
+      value: withsecure_elements
+  - set:
+      field: event.action
+      value: detected
+  - set:
+      field: event.outcome
+      value: success
+  - set:
+      field: event.id
+      value: "{{id}}"
+      if: ctx.id != null
+      tag: set_event_id
+  - set:
+      field: event.created
+      value: "{{createdTimestamp}}"
+      if: ctx.createdTimestamp != null
+      tag: set_event_created
+  - set:
+      field: event.start
+      value: "{{createdTimestamp}}"
+      if: ctx.createdTimestamp != null
+      tag: set_event_start
+  - set:
+      field: event.end
+      value: "{{updatedTimestamp}}"
+      if: ctx.updatedTimestamp != null
+      tag: set_event_end
+  - set:
+      field: event.severity
+      value: "{{severity}}"
+      if: ctx.severity != null
+      tag: set_event_severity
+
+  # Set WithSecure custom fields
+  - set:
+      field: withsecure.incident.detection.id
+      value: "{{id}}"
+      if: ctx.id != null
+      ignore_failure: true
+  - set:
+      field: withsecure.incident.detection.incident_id
+      value: "{{incidentId}}"
+      if: ctx.incidentId != null
+      ignore_failure: true
+  - set:
+      field: withsecure.incident.detection.detection_type
+      value: "{{detectionType}}"
+      if: ctx.detectionType != null
+      ignore_failure: true
+  - set:
+      field: withsecure.incident.detection.severity
+      value: "{{severity}}"
+      if: ctx.severity != null
+      ignore_failure: true
+  - set:
+      field: withsecure.incident.detection.status
+      value: "{{status}}"
+      if: ctx.status != null
+      ignore_failure: true
+  - date:
+      field: createdTimestamp
+      target_field: withsecure.incident.detection.created_timestamp
+      formats:
+        - ISO8601
+        - UNIX_MS
+      if: ctx.createdTimestamp != null
+      ignore_failure: true
+      tag: convert_created_timestamp
+  - date:
+      field: updatedTimestamp
+      target_field: withsecure.incident.detection.updated_timestamp
+      formats:
+        - ISO8601
+        - UNIX_MS
+      if: ctx.updatedTimestamp != null
+      ignore_failure: true
+      tag: convert_updated_timestamp
+  - set:
+      field: withsecure.incident.detection.source
+      value: "{{source}}"
+      if: ctx.source != null
+      ignore_failure: true
+  - set:
+      field: withsecure.incident.detection.category
+      value: "{{category}}"
+      if: ctx.category != null
+      ignore_failure: true
+  - set:
+      field: withsecure.incident.detection.description
+      value: "{{description}}"
+      if: ctx.description != null
+      ignore_failure: true
+  - rename:
+      field: details
+      target_field: withsecure.incident.detection.details
+      ignore_missing: true
+  - rename:
+      field: device
+      target_field: withsecure.incident.detection.device
+      ignore_missing: true
+  - rename:
+      field: user
+      target_field: withsecure.incident.detection.user
+      ignore_missing: true
+  - rename:
+      field: file
+      target_field: withsecure.incident.detection.file
+      ignore_missing: true
+  - rename:
+      field: network
+      target_field: withsecure.incident.detection.network
+      ignore_missing: true
+
+  # Clean up temporary fields
+  - remove:
+      field:
+        - id
+        - incidentId
+        - detectionType
+        - severity
+        - status
+        - createdTimestamp
+        - updatedTimestamp
+        - source
+        - category
+        - description
+      ignore_missing: true
+
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'

packages/withsecure_elements/data_stream/incident_detections/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: "@timestamp"
+  type: date
+  description: Event timestamp.

packages/withsecure_elements/data_stream/incident_detections/fields/ecs.yml

@@ -0,0 +1,79 @@
+- name: event
+  type: group
+  description: Event fields
+  fields:
+    - name: category
+      type: keyword
+      description: Event category
+    - name: type
+      type: keyword
+      description: Event type
+    - name: dataset
+      type: keyword
+      description: Dataset name
+    - name: module
+      type: keyword
+      description: Module name
+    - name: severity
+      type: long
+      description: Event severity
+    - name: original
+      type: keyword
+      description: Original event message
+    - name: id
+      type: keyword
+      description: Event ID
+    - name: action
+      type: keyword
+      description: Event action
+    - name: kind
+      type: keyword
+      description: Event kind
+    - name: outcome
+      type: keyword
+      description: Event outcome
+    - name: provider
+      type: keyword
+      description: Event provider
+    - name: code
+      type: keyword
+      description: Event code
+    - name: reason
+      type: keyword
+      description: Event reason
+    - name: sequence
+      type: long
+      description: Event sequence number
+    - name: created
+      type: date
+      description: Event creation time
+    - name: start
+      type: date
+      description: Event start time
+    - name: end
+      type: date
+      description: Event end time
+    - name: duration
+      type: long
+      description: Event duration
+    - name: timezone
+      type: keyword
+      description: Event timezone
+    - name: url
+      type: keyword
+      description: Event URL
+    - name: reference
+      type: keyword
+      description: Event reference
+    - name: hash
+      type: keyword
+      description: Event hash
+    - name: risk_score
+      type: float
+      description: Event risk score
+    - name: risk_score_norm
+      type: float
+      description: Normalized event risk score
+    - name: ingested
+      type: date
+      description: Event ingestion time