[kubernetes] adds beta support for rotated logs

Enables the collection of rotated log files, including GZIP-compressed files, for Kubernetes container logs.

This enhancement introduces new configuration options to control the ingestion of rotated logs, including specifying paths for the rotated logs and enabling GZIP decompression.

Author: AndersonQ

packages/kubernetes/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.82.0"
+  changes:
+    - description: Add beta support to collect rotated (including GZIP) logs.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15441
 - version: "1.81.0"
   changes:
     - description: Support for collecting audit logs from cloud providers.

packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs

@@ -14,6 +14,15 @@ paths:
 {{#each paths}}
   - {{this}}
 {{/each}}
+{{#if rotated_logs}}
+  {{#each rotated_logs_paths}}
+  - {{this}}
+  {{/each}}
+{{/if}}
+
+{{#if rotated_logs}}
+gzip_experimental: true
+{{/if}}
 data_stream:
   dataset: {{data_stream.dataset}}
 prospector.scanner.symlinks: {{ symlinks }}

packages/kubernetes/data_stream/container_logs/manifest.yml

@@ -6,9 +6,25 @@ streams:
     title: Collect Kubernetes container logs
     description: Collect Kubernetes container logs
     vars:
+      - name: symlinks
+        type: bool
+        title: Use Symlinks
+        multi: false
+        required: true
+        show_user: true
+        default: true
+      - name: data_stream.dataset
+        required: true
+        default: kubernetes.container_logs
+        title: Dataset name
+        description: >
+          Set the name for your dataset. Changing the dataset will send the data to a different index. For more info look at [data_stream field](https://www.elastic.co/guide/en/ecs/master/ecs-data_stream.html).
+        type: text
+
       - name: paths
         type: text
         required: true
+        show_user: false
         title: Kubernetes container log path
         multi: true
         default:
@@ -20,21 +36,32 @@ streams:
           the paths contain no variable! Refer to the [integration
           documentation](https://www.elastic.co/guide/en/integrations/current/kubernetes.html)
           for more details.
-      - name: symlinks
+
+      - name: rotated_logs
         type: bool
-        title: Use Symlinks
+        title: Include rotated log files
         multi: false
+        required: false
+        show_user: false
+        default: false
+        description: |
+          Warning: it might lead to data re-ingestion. When enabled, Elastic-Agent will also ingest the rotated log files for each pod, including GZIP-compressed files (.gz), decompressing them as they are read. This is beta feature. For full details about GZIP decompression, see the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream#reading-gzip-files). Available for Elastic Agent 9.2.0 and newer.
+      - name: rotated_logs_paths
+        type: text
         required: true
-        show_user: true
-        default: true
-      - name: data_stream.dataset
-        required: true
-        default: kubernetes.container_logs
-        title: Dataset name
-        description: >
-          Set the name for your dataset. Changing the dataset will send the data to a different index. For more info look at [data_stream field](https://www.elastic.co/guide/en/ecs/master/ecs-data_stream.html).
+        show_user: false
+        title: Kubernetes container rotated log path
+        multi: true
+        default:
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log.*
+        description: >-
+          For every container the Elastic-Agent can see (usually every
+          container on the node) an instance of the input will be
+          created harvesting all paths defined here, even if 
+          the paths contain no variable! Refer to the [integration
+          documentation](https://www.elastic.co/guide/en/integrations/current/kubernetes.html)
+          for more details.
 
-        type: text
       - name: containerParserStream
         type: text
         title: Container parser's stream configuration

packages/kubernetes/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.1.2
 name: kubernetes
 title: Kubernetes
-version: 1.81.0
+version: 1.82.0
 description: Collect logs and metrics from Kubernetes clusters with Elastic Agent.
 type: integration
 categories: