Skip to content

Commit 7ba614d

Browse files
chemamartinezchemamartinez
authored
checkpoint_harmony_endpoint: improve handling of 404 and 503 errors (elastic#13009)
Changes in the error handling: - When receiving a 503 Service Unavailable response, the sequence is restarted gratefully, cleaning the task ID and page token and waiting for the next interval. - When receiving a 404 Not Found, the task ID is requested again for the same timeframe.
1 parent df51cde commit 7ba614d

File tree

9 files changed

+733
-40
lines changed

9 files changed

+733
-40
lines changed

‎packages/checkpoint_harmony_endpoint/changelog.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,12 @@
11
# newer versions go on top
2+
- version: "0.5.0"
3+
changes:
4+
- description: Improve handling of 404 and 503 API responses.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/13009
7+
- description: Propagate forensics CEL fixes to all data streams.
8+
type: enhancement
9+
link: https://github.com/elastic/integrations/pull/13009
210
- version: "0.4.0"
311
changes:
412
- description: Update Kibana constraint to support 9.0.0.

‎packages/checkpoint_harmony_endpoint/data_stream/antibot/agent/stream/cel.yml.hbs‎

Lines changed: 112 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,7 @@ state:
1414
limit: {{limit}}
1515
page_limit: {{page_limit}}
1616
filter: {{filter}}
17-
program: |-
18-
17+
program: |
1918
(
2019
state.?cursor.auth_data.expires.optMap(t,
2120
t.parse_time(time_layout.RFC1123) - now() > duration("5m")
@@ -35,22 +34,44 @@ program: |-
3534
"accessKey": state.auth_access_key,
3635
}.encode_json(),
3736
}
38-
).do_request().as(resp,
37+
).do_request().as(resp, resp.StatusCode == 200 ?
3938
bytes(resp.Body).decode_json().as(body,
4039
{
4140
"token": body.data.token,
4241
"expires": body.data.expires,
4342
}
4443
)
44+
:
45+
state.with(
46+
{
47+
"events": {
48+
"error": {
49+
"code": string(resp.StatusCode),
50+
"id": string(resp.Status),
51+
"message": "POST " + state.url.trim_right("/") + "/auth/external: " + (
52+
size(resp.Body) != 0 ?
53+
string(resp.Body)
54+
:
55+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
56+
),
57+
},
58+
},
59+
"want_more": false,
60+
}
61+
)
4562
)
46-
).as(auth_data,
63+
).as(v, has(v.?events.error) ?
64+
v
65+
: v.as(auth_data,
4766
(state.?cursor.task_id.orValue(null) == null) ?
4867
// No task ID - Submit a query and get its task ID.
4968
{
5069
"startTime": state.?cursor.next_startTime.orValue(
5170
timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339)
5271
),
53-
"endTime": timestamp(now() - duration("1m")).format(time_layout.RFC3339),
72+
"endTime": state.?cursor.next_endTime.orValue(
73+
timestamp(now() - duration("1m")).format(time_layout.RFC3339)
74+
),
5475
}.as(timeframe,
5576
request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query").with(
5677
{
@@ -97,7 +118,10 @@ program: |-
97118
"auth_data": auth_data,
98119
"task_id": body.data.taskId,
99120
"page_token": null,
121+
"current_startTime": timeframe.startTime,
122+
"current_endTime": timeframe.endTime,
100123
"next_startTime": timeframe.endTime,
124+
"next_endTime": null,
101125
},
102126
}
103127
)
@@ -125,6 +149,24 @@ program: |-
125149
}
126150
)
127151
:
152+
(resp.StatusCode == 404) ?
153+
// 404 Not Found - Resubmit the task ID query for the same timeframe.
154+
state.with(
155+
{
156+
"events": [{"message": {"event": {"reason": "polling"}}.encode_json()}],
157+
"want_more": true,
158+
"cursor": state.cursor.with(
159+
{
160+
"auth_data": auth_data,
161+
"task_id": null,
162+
"next_startTime": state.cursor.current_startTime,
163+
"next_endTime": state.cursor.current_endTime,
164+
}
165+
),
166+
}
167+
)
168+
:
169+
(resp.StatusCode == 200) ?
128170
bytes(resp.Body).decode_json().as(body,
129171
(body.data.state == "Ready") ?
130172
// 'Ready' (Results found) - Save the first page token.
@@ -168,6 +210,24 @@ program: |-
168210
}
169211
)
170212
)
213+
:
214+
state.with(
215+
{
216+
"events": {
217+
"error": {
218+
"code": string(resp.StatusCode),
219+
"id": string(resp.Status),
220+
"message": "GET " + state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query: " + (
221+
size(resp.Body) != 0 ?
222+
string(resp.Body)
223+
:
224+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
225+
),
226+
},
227+
},
228+
"want_more": false,
229+
}
230+
)
171231
)
172232
:
173233
// Task is ready - Use the task ID and page token to retrieve a page of results.
@@ -194,6 +254,34 @@ program: |-
194254
}
195255
)
196256
:
257+
(resp.StatusCode == 503) ?
258+
// 503 Service Unavailable - Clear the task ID and page token, and end the sequence.
259+
state.with(
260+
{
261+
"events": {
262+
"error": {
263+
"code": string(resp.StatusCode),
264+
"id": string(resp.Status),
265+
"message": "POST " + state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve: " + (
266+
size(resp.Body) != 0 ?
267+
string(resp.Body)
268+
:
269+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
270+
),
271+
},
272+
},
273+
"want_more": false,
274+
"cursor": state.cursor.with(
275+
{
276+
"auth_data": auth_data,
277+
"task_id": null,
278+
"page_token": null,
279+
}
280+
),
281+
}
282+
)
283+
:
284+
(resp.StatusCode == 200) ?
197285
bytes(resp.Body).decode_json().as(body,
198286
(body.data.nextPageToken != "NULL") ?
199287
// Not the last page - Save the next page token and continue.
@@ -225,8 +313,26 @@ program: |-
225313
}
226314
)
227315
)
316+
:
317+
state.with(
318+
{
319+
"events": {
320+
"error": {
321+
"code": string(resp.StatusCode),
322+
"id": string(resp.Status),
323+
"message": "POST " + state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve: " + (
324+
size(resp.Body) != 0 ?
325+
string(resp.Body)
326+
:
327+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
328+
),
329+
},
330+
},
331+
"want_more": false,
332+
}
333+
)
228334
)
229-
)
335+
))
230336
tags:
231337
{{#if preserve_original_event}}
232338
- preserve_original_event

‎packages/checkpoint_harmony_endpoint/data_stream/antimalware/agent/stream/cel.yml.hbs‎

Lines changed: 112 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,7 @@ state:
1414
limit: {{limit}}
1515
page_limit: {{page_limit}}
1616
filter: {{filter}}
17-
program: |-
18-
17+
program: |
1918
(
2019
state.?cursor.auth_data.expires.optMap(t,
2120
t.parse_time(time_layout.RFC1123) - now() > duration("5m")
@@ -35,22 +34,44 @@ program: |-
3534
"accessKey": state.auth_access_key,
3635
}.encode_json(),
3736
}
38-
).do_request().as(resp,
37+
).do_request().as(resp, resp.StatusCode == 200 ?
3938
bytes(resp.Body).decode_json().as(body,
4039
{
4140
"token": body.data.token,
4241
"expires": body.data.expires,
4342
}
4443
)
44+
:
45+
state.with(
46+
{
47+
"events": {
48+
"error": {
49+
"code": string(resp.StatusCode),
50+
"id": string(resp.Status),
51+
"message": "POST " + state.url.trim_right("/") + "/auth/external: " + (
52+
size(resp.Body) != 0 ?
53+
string(resp.Body)
54+
:
55+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
56+
),
57+
},
58+
},
59+
"want_more": false,
60+
}
61+
)
4562
)
46-
).as(auth_data,
63+
).as(v, has(v.?events.error) ?
64+
v
65+
: v.as(auth_data,
4766
(state.?cursor.task_id.orValue(null) == null) ?
4867
// No task ID - Submit a query and get its task ID.
4968
{
5069
"startTime": state.?cursor.next_startTime.orValue(
5170
timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339)
5271
),
53-
"endTime": timestamp(now() - duration("1m")).format(time_layout.RFC3339),
72+
"endTime": state.?cursor.next_endTime.orValue(
73+
timestamp(now() - duration("1m")).format(time_layout.RFC3339)
74+
),
5475
}.as(timeframe,
5576
request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query").with(
5677
{
@@ -97,7 +118,10 @@ program: |-
97118
"auth_data": auth_data,
98119
"task_id": body.data.taskId,
99120
"page_token": null,
121+
"current_startTime": timeframe.startTime,
122+
"current_endTime": timeframe.endTime,
100123
"next_startTime": timeframe.endTime,
124+
"next_endTime": null,
101125
},
102126
}
103127
)
@@ -125,6 +149,24 @@ program: |-
125149
}
126150
)
127151
:
152+
(resp.StatusCode == 404) ?
153+
// 404 Not Found - Resubmit the task ID query for the same timeframe.
154+
state.with(
155+
{
156+
"events": [{"message": {"event": {"reason": "polling"}}.encode_json()}],
157+
"want_more": true,
158+
"cursor": state.cursor.with(
159+
{
160+
"auth_data": auth_data,
161+
"task_id": null,
162+
"next_startTime": state.cursor.current_startTime,
163+
"next_endTime": state.cursor.current_endTime,
164+
}
165+
),
166+
}
167+
)
168+
:
169+
(resp.StatusCode == 200) ?
128170
bytes(resp.Body).decode_json().as(body,
129171
(body.data.state == "Ready") ?
130172
// 'Ready' (Results found) - Save the first page token.
@@ -168,6 +210,24 @@ program: |-
168210
}
169211
)
170212
)
213+
:
214+
state.with(
215+
{
216+
"events": {
217+
"error": {
218+
"code": string(resp.StatusCode),
219+
"id": string(resp.Status),
220+
"message": "GET " + state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query: " + (
221+
size(resp.Body) != 0 ?
222+
string(resp.Body)
223+
:
224+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
225+
),
226+
},
227+
},
228+
"want_more": false,
229+
}
230+
)
171231
)
172232
:
173233
// Task is ready - Use the task ID and page token to retrieve a page of results.
@@ -194,6 +254,34 @@ program: |-
194254
}
195255
)
196256
:
257+
(resp.StatusCode == 503) ?
258+
// 503 Service Unavailable - Clear the task ID and page token, and end the sequence.
259+
state.with(
260+
{
261+
"events": {
262+
"error": {
263+
"code": string(resp.StatusCode),
264+
"id": string(resp.Status),
265+
"message": "POST " + state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve: " + (
266+
size(resp.Body) != 0 ?
267+
string(resp.Body)
268+
:
269+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
270+
),
271+
},
272+
},
273+
"want_more": false,
274+
"cursor": state.cursor.with(
275+
{
276+
"auth_data": auth_data,
277+
"task_id": null,
278+
"page_token": null,
279+
}
280+
),
281+
}
282+
)
283+
:
284+
(resp.StatusCode == 200) ?
197285
bytes(resp.Body).decode_json().as(body,
198286
(body.data.nextPageToken != "NULL") ?
199287
// Not the last page - Save the next page token and continue.
@@ -225,8 +313,26 @@ program: |-
225313
}
226314
)
227315
)
316+
:
317+
state.with(
318+
{
319+
"events": {
320+
"error": {
321+
"code": string(resp.StatusCode),
322+
"id": string(resp.Status),
323+
"message": "POST " + state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve: " + (
324+
size(resp.Body) != 0 ?
325+
string(resp.Body)
326+
:
327+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
328+
),
329+
},
330+
},
331+
"want_more": false,
332+
}
333+
)
228334
)
229-
)
335+
))
230336
tags:
231337
{{#if preserve_original_event}}
232338
- preserve_original_event

0 commit comments

Comments
 (0)