[github] audit: prefer @timestamp over created_at in agent cursor logic (elastic#16097)

Although the query parameter is `created`, listings are ordered by
`@timestamp`. The `created_at` field often has older times, and it's
absent from some even types, such as git events[1].

We keep `created_at` as a fallback, because some events don't have
`@timestamp` according to the documentation[2] (although that hasn't
been verified in the live API).

[1]: https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/audit-log-events-for-your-enterprise#git
[2]: https://docs.github.com/en/enterprise-cloud@latest/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/audit-log-events-for-your-enterprise#code_scanning

Author: chrisberkhout

packages/github/_dev/deploy/docker/files/config.yml

@@ -35,7 +35,6 @@ rules:
             "@timestamp": 1605719148837,
             "action": "repo.destroy",
             "actor": "monalisa",
-            "created_at": 1605719148837,
             "_document_id": "LwW2vpJZCDS-WUmo9Z-ifw",
             "org": "mona-org",
             "repo": "mona-org/mona-test-repo",
@@ -70,7 +69,6 @@ rules:
             "_document_id": "Vqvg6kZ4MYqwWRKFDzlMoQ",
             "org": "octocat-test-org"
           },{
-            "@timestamp": 1605719148837,
             "action": "repo.destroy",
             "actor": "monalisa",
             "created_at": 1605719148837,
@@ -112,7 +110,6 @@ rules:
             "_document_id": "Vqvg6kZ4MYqwWRKFDzlMoQ",
             "org": "octocat-test-org"
           },{
-            "@timestamp": 1605719148837,
             "action": "repo.destroy",
             "actor": "monalisa",
             "created_at": 1605719148837,
@@ -153,7 +150,6 @@ rules:
             "@timestamp": 1605719148837,
             "action": "repo.destroy",
             "actor": "monalisa",
-            "created_at": 1605719148837,
             "_document_id": "LwW2vpJZCDS-WUmo9Z-ifw",
             "org": "mona-org",
             "repo": "mona-org/mona-test-repo",

packages/github/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.17.3"
+  changes:
+    - description: Fix HTTPJSON cursor logic for audit data stream.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/16097
 - version: "2.17.2"
   changes:
     - description: Remove updated_at field from latest issues transform unique keys.

packages/github/data_stream/audit/agent/stream/httpjson.yml.hbs

@@ -55,7 +55,12 @@ response.pagination:
 
 cursor:
   last_timestamp:
-    value: '[[ .last_event.created_at ]]'
+    value: >-
+      [[- if index .last_event "@timestamp" -]]
+        [[- .last_event.Get "@timestamp" -]]
+      [[- else -]]
+        [[- .last_event.created_at -]]
+      [[- end -]]
     fail_on_template_error: true
 
 {{#if tags.length}}
@@ -76,4 +81,4 @@ publisher_pipeline.disable_host: true
 {{#if processors}}
 processors:
 {{processors}}
-{{/if}}
+{{/if}}

packages/github/manifest.yml

@@ -1,6 +1,6 @@
 name: github
 title: GitHub
-version: "2.17.2"
+version: "2.17.3"
 description: Collect logs from GitHub with Elastic Agent.
 type: integration
 format_version: "3.4.0"