ti_abusech: Handle API errors inside CEL (#13708)

ti_abusech: Report API errors inside CEL programs.

Currently some of the CEL programs inside ti_abusech don't check if 
API returns success. There are also certain checks missing on empty
data. These lead to CEL execution errors. 
Add checks where necessary and report errors inside all data-streams.

Also update remove the conversion of bytes for resp.Body as it is 
no longer required after 8.16.

Author: kcreddy

packages/ti_abusech/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.9.1"
+  changes:
+    - description: Properly handle CEL errors.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/13708
 - version: "2.9.0"
   changes:
     - description: Enable request trace log removal.

packages/ti_abusech/data_stream/malware/agent/stream/cel.yml.hbs

@@ -31,13 +31,33 @@ program: |
         :
           optional.none(),
       }
-    }).as(req, req.do_request().as(resp, 
-      bytes(resp.Body).decode_json().as(body, {
-        "events": body.payloads.map(payload, {
-          "message": payload.encode_json()
-        }),
+    }).as(req, req.do_request().as(resp, resp.StatusCode == 200 ?
+      resp.Body.decode_json().as(body, {
+        "events": (has(body.payloads) ?
+          body.payloads.map(payload, {
+            "message": payload.encode_json()
+          })
+        :
+          []
+        ),
         "url": state.url
       })
+    :
+      {
+        "events": {
+          "error": {
+            "code": string(resp.StatusCode),
+            "id": string(resp.Status),
+            "message": "GET "+ state.url.trim_right("/") + ":" + (
+              size(resp.Body) != 0 ?
+                string(resp.Body)
+              :
+                string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+            ),
+          },
+        },
+        "want_more": false,
+      }
     ))
   )
 

packages/ti_abusech/data_stream/malwarebazaar/agent/stream/cel.yml.hbs

@@ -32,7 +32,7 @@ program: |
           optional.none(),
       }
     }).do_request().as(resp, resp.StatusCode == 200 ?
-      bytes(resp.Body).decode_json().as(body, body.?query_status == optional.of("ok") ?
+      resp.Body.decode_json().as(body, body.?query_status == optional.of("ok") ?
         {
           "events": body.data.map(ind, {
             "message": ind.encode_json()

packages/ti_abusech/data_stream/threatfox/agent/stream/cel.yml.hbs

@@ -33,17 +33,37 @@ program: |
           :
             optional.none(),
           }
-    }).as(req, req.do_request().as(resp, 
-      bytes(resp.Body).decode_json().as(body, {
-        "events": body.data.map(ind, {
-          "message": ind.encode_json()
-        }),
+    }).as(req, req.do_request().as(resp, resp.StatusCode == 200 ?
+      resp.Body.decode_json().as(body, {
+        "events": (has(body.data) ?
+          body.data.map(ind, {
+            "message": ind.encode_json()
+          })
+        :
+          []
+        ),
         "cursor": {
           "days": "1"
         },
         "initial_interval": state.initial_interval,
         "url": state.url
       })
+    :
+      {
+        "events": {
+          "error": {
+            "code": string(resp.StatusCode),
+            "id": string(resp.Status),
+            "message": "POST "+ state.url.trim_right("/") + ":" + (
+              size(resp.Body) != 0 ?
+                string(resp.Body)
+              :
+                string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+            ),
+          },
+        },
+        "want_more": false,
+      }
     ))
   )
 

packages/ti_abusech/data_stream/url/agent/stream/cel.yml.hbs

@@ -33,21 +33,39 @@ program: |
       }
     })
     .do_request()
-    .as(resp, resp.Body.mime("application/zip").File.as(file, file.size() > 0 ? 
-      file[0].Data.as(data, bytes(data).decode_json().as(body, {
-        "events": body.map(id, body[id].size() > 0 ? 
-          {"message": body[id][0].with({"id": id}).encode_json()}
-        :
-          {"message": ""}
-        ),
-        "url": state.url
-      }))
+    .as(resp, resp.StatusCode == 200 ?
+      resp.Body.mime("application/zip").File.as(file, file.size() > 0 ? 
+        file[0].Data.as(data, bytes(data).decode_json().as(body, {
+          "events": body.map(id, body[id].size() > 0 ? 
+            {"message": body[id][0].with({"id": id}).encode_json()}
+          :
+            {"message": ""}
+          ),
+          "url": state.url
+        }))
+      :
+        {
+          "events": [],
+          "url": state.url
+        }
+      )
     :
       {
-        "events": [],
-        "url": state.url
+        "events": {
+          "error": {
+            "code": string(resp.StatusCode),
+            "id": string(resp.Status),
+            "message": "GET "+ state.url.trim_right("/") + ":" + (
+              size(resp.Body) != 0 ?
+                string(resp.Body)
+              :
+                string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+            ),
+          },
+        },
+        "want_more": false,
       }
-    ))
+    )
   )
 
 fields_under_root: true

packages/ti_abusech/manifest.yml

@@ -1,6 +1,6 @@
 name: ti_abusech
 title: AbuseCH
-version: "2.9.0"
+version: "2.9.1"
 description: Ingest threat intelligence indicators from URL Haus, Malware Bazaar, and Threat Fox feeds with Elastic Agent.
 type: integration
 format_version: "3.2.3"