Skip to content

Commit a249e6c

Browse files
eset_protect: Fix parsing of parsing HEARTBEAT messages, Enhance error handling for agent failures
ESET sends heartbeat messages every 1–3 minutes to keep the connection open. We collect those events and drop them from the ingest pipeline. Improve error handling and display for CEL code, and use the terminate processor to handle agent errors in the ingest pipeline. Update format_version to 3.3.2. Bump the minimum stack version to ^8.16.0 || ^9.0.0.
1 parent 77f1765 commit a249e6c

File tree

18 files changed

+181
-98
lines changed

18 files changed

+181
-98
lines changed
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
<15>Oct 30 05:31:10 co7 ERAServer[75]: {"event_type":"FilteredWebsites_Event","ipv4":"192.168.30.30","hostname":"win-test","group_name":"All/Lost & found","os_name":"Microsoft Windows 11 Pro","group_description":"Lost & found static group","source_uuid":"d9477661-8fa4-4144-b8d4-e37b983bcd69","occured":"21-Jun-2021 03:56:20","severity":"Warning","event":"An attempt to connect to URL","target_address":"89.160.20.128","target_address_type":"IPv4","scanner_id":"HTTP filter","action_taken":"blocked","object_uri":"https://test.com","hash":"ABCDAA625E6961037B8904E113FD0C232A7D0EDC","username":"WIN-TEST\\Administrator","processname":"C:\\Program Files\\Web browser\\brwser.exe","rule_id":"Blocked by PUA blacklist"}
2+
<15>Oct 30 05:31:10 co7 ERAServer[75]: HEARTBEAT

packages/eset_protect/changelog.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,12 @@
11
# newer versions go on top
2+
- version: "1.10.0"
3+
changes:
4+
- description: "Enhance error handling for agent failures."
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/15799
7+
- description: "Fix parsing of HEARTBEAT messages in event data stream."
8+
type: bugfix
9+
link: https://github.com/elastic/integrations/pull/15799
210
- version: "1.9.0"
311
changes:
412
- description: "Fix issue with missing parameter mapping causing 'invalid_grant' error during OAuth2 username password authentication."

packages/eset_protect/data_stream/detection/_dev/test/pipeline/test-detection.log

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,3 @@
55
{"category":"DETECTION_CATEGORY_UNSPECIFIED","typeName":"Test file","objectUrl":"C:\\Temp\\06516f11-xxxx-xxxx-xxxx-37da66b5de99_ccf7464ba6e2e12e984514f694bfb10d03de77358d8a3afd7a2ffed150ec1df8.zip.e99\\ccf7464ba6e2e12e984514f694bfb10d03de77358d8a3afd7a2ffed150ec1df8","uuid":"xxx-xxxx-9789-1234-xxxxxxxxxxxx","severityLevel":"SEVERITY_LEVEL_UNSPECIFIED","responses":[{}],"occurTime":"2023-07-28T10:36:53Z","objectTypeName":"Memory","objectHashSha1":"AAF4C61DDCC5E8A2DABEDE0F3B4820123456780D","networkCommunication":{"protocolName":"0","remoteIpAddress":"175.16.199.1","remotePort":8080,"localIpAddress":"216.160.83.56","localPort":80,"direction":"NETWORK_COMMUNICATION_DIRECTION_OUTBOUND"},"context":{"process":{"path":"C:\\exampletest.exe"},"deviceUuid":"xxx-1234-1234-1234-xxxxxxxxxxxx","userName":"example\\exmpleuser","circumstances":"Win32/Injector.DTLK"}}
66
{"category":"DETECTION_CATEGORY_HIPS_RULE","typeName":"Potentially unwanted application","objectUrl":"C:\\Temp\\06516f11-xxxx-xxxx-xxxx-37da66b5de99_ccf7464ba6e2e12e984514f694bfb10d03de77358d8a3afd7a2ffed150ec1df8.zip.e99\\ccf7464ba6e2e12e984514f694bfb10d03de77358d8a3afd7a2ffed150ec1df8","uuid":"xxx-1234-9789-1234-xxxxxxxxxxxx","severityLevel":"SEVERITY_LEVEL_DIAGNOSTIC","responses":[{}],"occurTime":"2023-07-15T10:36:53Z","objectTypeName":"Memory","objectHashSha1":"AAF4C61DDCC5E8A2DABEDE0F3B4820123456780D","networkCommunication":{"protocolName":"http","remoteIpAddress":"1.128.0.0","remotePort":56,"localIpAddress":"1.128.0.5","localPort":8080,"direction":"NETWORK_COMMUNICATION_DIRECTION_UNSPECIFIED"},"context":{"process":{"path":"C:\\exampletest.exe"},"deviceUuid":"xxx-1234-2345-1234-xxxxxxxxxxxx","userName":"example","circumstances":"Blocked by Administrator"}}
77
{"category":"DETECTION_CATEGORY_UNSPECIFIED","context":{"circumstances":"Malware: VBS/TrojanDownloader.Agent.YUI","deviceUuid":"16b429cb-c064-4a31-98ba-62fff54f0c96","process":{"path":"%SYSTEM%\\windowspowershell\\v1.0\\powershell.exe"},"userName":"kate-ebademo\\kate"},"displayName":"","networkCommunication":{"direction":"NETWORK_COMMUNICATION_DIRECTION_UNSPECIFIED","localIpAddress":"","localPort":0,"protocolName":"0","remoteIpAddress":"","remotePort":0},"objectHashSha1":"22B9B35A804A7A3739CBD007E00959075AECF0FC","objectName":"","objectTypeName":"File","objectUrl":"script","occurTime":"2024-03-27T09:54:20Z","responses":[{"description":"","deviceRestartRequired":false,"displayName":"","protectionName":""}],"severityLevel":"SEVERITY_LEVEL_LOW","typeName":"nil","uuid":"ae4d218f-5806-e446-0b86-609e5a4cfa94"}
8-
{"message":"retry"}

packages/eset_protect/data_stream/detection/_dev/test/pipeline/test-detection.log-expected.json

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -837,7 +837,6 @@
837837
"domain": "kate-ebademo",
838838
"name": "kate"
839839
}
840-
},
841-
null
840+
}
842841
]
843842
}

packages/eset_protect/data_stream/detection/agent/stream/cel.yml.hbs

Lines changed: 40 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ program: |
5252
).as(state,
5353
request(
5454
"GET",
55-
state.url + "/v1/detections?" + {
55+
state.url.trim_right("/") + "/v1/detections?" + {
5656
"page_size": [state.page_size],
5757
"start_time": [state.start_time],
5858
"end_time": [state.end_time],
@@ -72,7 +72,7 @@ program: |
7272
)).do_request().as(resp, (
7373
resp.StatusCode == 200
7474
?
75-
bytes(resp.Body).decode_json().as(body, {
75+
resp.Body.decode_json().as(body, {
7676
"events": (
7777
has(body.detections) && body.detections != null
7878
?
@@ -91,6 +91,7 @@ program: |
9191
),
9292
"want_more": has(body.nextPageToken) && body.nextPageToken != null && body.nextPageToken != "",
9393
"page_size": state.page_size,
94+
"initial_interval": state.initial_interval,
9495
"start_time": string(state.start_time),
9596
"end_time": string(state.end_time),
9697
"cursor": {
@@ -115,18 +116,38 @@ program: |
115116
}
116117
})
117118
:
118-
{
119-
"events": resp.StatusCode == 202 ? [{"message":"retry"}] : [],
120-
"page_size": state.page_size,
121-
"page_token": resp.StatusCode == 202 ? state.page_token : "",
122-
"start_time": state.start_time,
123-
"end_time": state.end_time,
124-
"want_more": resp.StatusCode == 202 ? true : false,
125-
"cursor": {
126-
"last_timestamp": state.start_time,
127-
"response_id": resp.StatusCode == 202 ? resp.Header["Response-Id"][0] : null
128-
}
129-
}
119+
(
120+
resp.StatusCode == 202 ?
121+
state.with({
122+
"events": [{"message":"retry"}],
123+
"want_more": true,
124+
"cursor": {
125+
"last_timestamp": state.start_time,
126+
"response_id": resp.Header["Response-Id"][0]
127+
}
128+
})
129+
:
130+
state.with({
131+
"events": {
132+
"error": {
133+
"code": string(resp.StatusCode),
134+
"id": string(resp.Status),
135+
"message": "GET " + state.url.trim_right("/") + "/v1/detections :" + (
136+
size(resp.Body) != 0 ?
137+
string(resp.Body)
138+
:
139+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
140+
),
141+
},
142+
},
143+
"page_token": "",
144+
"want_more": false,
145+
"cursor": {
146+
"last_timestamp": state.start_time,
147+
"response_id": null
148+
}
149+
})
150+
)
130151
))
131152
)
132153
tags:
@@ -142,7 +163,11 @@ tags:
142163
{{#contains "forwarded" tags}}
143164
publisher_pipeline.disable_host: true
144165
{{/contains}}
145-
{{#if processors}}
146166
processors:
167+
- drop_event:
168+
when:
169+
equals:
170+
message: retry
171+
{{#if processors}}
147172
{{processors}}
148173
{{/if}}

packages/eset_protect/data_stream/detection/elasticsearch/ingest_pipeline/default.yml

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,23 @@ processors:
55
field: ecs.version
66
tag: set_ecs_version
77
value: 8.11.0
8+
- terminate:
9+
tag: data_collection_error
10+
if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null
11+
description: error message set and no data to process.
812
- rename:
913
field: message
1014
tag: rename_message_to_event_original
1115
target_field: event.original
1216
ignore_missing: true
17+
description: Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.
1318
if: ctx.event?.original == null
19+
- remove:
20+
field: message
21+
tag: remove_message
22+
ignore_missing: true
23+
description: The `message` field is no longer required if the document has an `event.original` field.
24+
if: ctx.event?.original != null
1425
- json:
1526
field: event.original
1627
tag: json_event_original
@@ -19,9 +30,6 @@ processors:
1930
- append:
2031
field: error.message
2132
value: "Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}"
22-
- drop:
23-
if: ctx.json?.message == "retry"
24-
tag: drop_retry_events
2533
- fingerprint:
2634
fields:
2735
- json.uuid

packages/eset_protect/data_stream/detection/sample_event.json

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
{
22
"@timestamp": "2023-10-26T13:36:53.000Z",
33
"agent": {
4-
"ephemeral_id": "a2da59f5-382d-41e2-be5e-0b06df998911",
5-
"id": "930b36c5-0fd6-41c4-83bc-d8547e3fa880",
6-
"name": "docker-fleet-agent",
4+
"ephemeral_id": "c57caace-7bf5-4540-8dbf-086080c70b5f",
5+
"id": "55ae3f57-aa54-4ac3-a016-4f0ad3b506c2",
6+
"name": "elastic-agent-24261",
77
"type": "filebeat",
8-
"version": "8.12.0"
8+
"version": "8.16.0"
99
},
1010
"data_stream": {
1111
"dataset": "eset_protect.detection",
12-
"namespace": "ep",
12+
"namespace": "83546",
1313
"type": "logs"
1414
},
1515
"destination": {
@@ -38,9 +38,9 @@
3838
"version": "8.11.0"
3939
},
4040
"elastic_agent": {
41-
"id": "930b36c5-0fd6-41c4-83bc-d8547e3fa880",
41+
"id": "55ae3f57-aa54-4ac3-a016-4f0ad3b506c2",
4242
"snapshot": false,
43-
"version": "8.12.0"
43+
"version": "8.16.0"
4444
},
4545
"eset_protect": {
4646
"detection": {
@@ -75,7 +75,7 @@
7575
"intrusion_detection"
7676
],
7777
"dataset": "eset_protect.detection",
78-
"ingested": "2024-04-16T05:41:07Z",
78+
"ingested": "2025-10-29T15:12:55Z",
7979
"kind": "alert",
8080
"original": "{\"category\":\"DETECTION_CATEGORY_NETWORK_INTRUSION\",\"context\":{\"circumstances\":\"Eicar\",\"deviceUuid\":\"xxx-xxxx-1234-5678-xxxxxxxxxxxx\",\"process\":{\"path\":\"C:\\\\Windows\\\\chrome.exe\"},\"userName\":\"testingpc\\\\example\"},\"networkCommunication\":{\"protocolName\":\"0\",\"remoteIpAddress\":\"89.160.20.112\",\"remotePort\":443},\"objectHashSha1\":\"AAF4C61DDCC5E8A2DABEDE0F3B4820123456789D\",\"objectTypeName\":\"File\",\"objectUrl\":\"C:\\\\Temp\\\\06516f11-xxxx-xxxx-xxxx-37da66b5de99_ccf7464ba6e2e12e984514f694bfb10d03de77358d8a3afd7a2ffed150ec1df8.zip.e99\\\\ccf7464ba6e2e12e984514f694bfb10d03de77358d8a3afd7a2ffed150ec1df8\",\"occurTime\":\"2023-10-26T13:36:53Z\",\"responses\":[{}],\"severityLevel\":\"SEVERITY_LEVEL_MEDIUM\",\"typeName\":\"TCP Port scanning attack\",\"uuid\":\"xxx-xxxx-xxxx-1234-xxxxxxxxxxxx\"}",
8181
"type": [
@@ -137,4 +137,4 @@
137137
"domain": "testingpc",
138138
"name": "example"
139139
}
140-
}
140+
}

packages/eset_protect/data_stream/device_task/_dev/test/pipeline/test-device-task.log

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,3 @@
33
{"uuid":"c93070e0-XXXX-XXXX-5678-c48f0e5e0b7e","action":{"params":{"customProfileName":"DefaultProfile","scanTargets":["eset://AllTargets"],"cleaningEnabled":true,"scanProfile":"InDepth","@type":"type.googleapis.com/Era.Common.DataDefinition.Task.ESS.OnDemandScan"},"name":"Upgrade Agent"},"targets":{"devicesUuids":["0205321e-XXXX-XXXX-1234-feeb35010ea7","0205321e-XXXX-XXXX-5678-feeb35010ea7"]},"displayName":"Upgrade Agent - via context menu","triggers":[{"manual":{"expireTime":"2023-08-01T15:30:00Z"}}],"versionId":"1511","description":"Automatically created via context menu"}
44
{"uuid":"c93070e0-XXXX-1234-5678-c48f0e5e0b7e","action":{"params":{"customProfileName":"DefaultProfile","scanTargets":["eset://AllTargets"],"cleaningEnabled":true,"scanProfile":"InDepth","@type":"type.googleapis.com/Era.Common.DataDefinition.Task.ESS.OnDemandScan"},"name":"Shutdown computer"},"targets":{"devicesUuids":["0205321e-XXXX-XXXX-1234-feeb35010ea7","0205321e-XXXX-XXXX-5678-feeb35010ea7","0205321e-XXXX-1234-5678-feeb35010ea7"]},"displayName":"Reboot Computer - via context menu","triggers":[{"manual":{"expireTime":"2023-12-01T01:30:00Z"}}],"versionId":"1511","description":"Automatically created via context menu"}
55
{"uuid":"c93070e0-1234-1234-5678-c48f0e5e0b7e","action":{"params":{"customProfileName":"DefaultProfile","scanTargets":["eset://AllTargets"],"cleaningEnabled":true,"scanProfile":"InDepth","@type":"type.googleapis.com/Era.Common.DataDefinition.Task.ESS.OnDemandScan"},"name":"Product Activation"},"targets":{"devicesUuids":["0205321e-XXXX-1234-5678-feeb35010ea7"]},"displayName":"Product activation - via ESET LiveGuard","triggers":[{"manual":{"expireTime":"2023-12-01T01:30:00Z"}}],"versionId":"1511","description":"Automatically created via ESET LiveGuard"}
6-
{"message":"retry"}

packages/eset_protect/data_stream/device_task/_dev/test/pipeline/test-device-task.log-expected.json

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,6 @@
303303
"preserve_original_event",
304304
"preserve_duplicate_custom_fields"
305305
]
306-
},
307-
null
306+
}
308307
]
309308
}

packages/eset_protect/data_stream/device_task/agent/stream/cel.yml.hbs

Lines changed: 37 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ program: |
3434
(
3535
request(
3636
"GET",
37-
state.url + "/v1/device_tasks?" + {
37+
state.url.trim_right("/") + "/v1/device_tasks?" + {
3838
"page_size": [string(state.page_size)],
3939
"page_token": [state.page_token]
4040
}.format_query()
@@ -52,7 +52,7 @@ program: |
5252
)).do_request().as(resp, (
5353
resp.StatusCode == 200
5454
?
55-
bytes(resp.Body).decode_json().as(body, {
55+
resp.Body.decode_json().as(body, {
5656
"events": (
5757
has(body.tasks) && body.tasks != null
5858
?
@@ -73,15 +73,36 @@ program: |
7373
"page_size": state.page_size
7474
})
7575
:
76-
{
77-
"events": resp.StatusCode == 202 ? [{"message":"retry"}] : [],
78-
"page_size": state.page_size,
79-
"page_token": resp.StatusCode == 202 ? state.page_token : "",
80-
"want_more": resp.StatusCode == 202 ? true : false,
81-
"cursor": {
82-
"response_id": resp.StatusCode == 202 ? resp.Header["Response-Id"][0] : null
83-
}
84-
}
76+
(
77+
resp.StatusCode == 202 ?
78+
state.with({
79+
"events": [{"message":"retry"}],
80+
"want_more": true,
81+
"cursor": {
82+
"response_id": resp.Header["Response-Id"][0]
83+
}
84+
})
85+
:
86+
state.with({
87+
"events": {
88+
"error": {
89+
"code": string(resp.StatusCode),
90+
"id": string(resp.Status),
91+
"message": "GET " + state.url.trim_right("/") + "/v1/device_tasks :" + (
92+
size(resp.Body) != 0 ?
93+
string(resp.Body)
94+
:
95+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
96+
),
97+
},
98+
},
99+
"page_token": "",
100+
"want_more": false,
101+
"cursor": {
102+
"response_id": null
103+
}
104+
})
105+
)
85106
))
86107
)
87108
tags:
@@ -97,7 +118,11 @@ tags:
97118
{{#contains "forwarded" tags}}
98119
publisher_pipeline.disable_host: true
99120
{{/contains}}
100-
{{#if processors}}
101121
processors:
122+
- drop_event:
123+
when:
124+
equals:
125+
message: retry
126+
{{#if processors}}
102127
{{processors}}
103128
{{/if}}

0 commit comments

Comments
 (0)