Add datastream lifecycle support to indices metadata (#245548)

## Summary

Add [Datastream life
cycle](https://www.elastic.co/docs/manage-data/lifecycle/data-stream)
support to the indices metadata plugin.

For data streams using DSL, the plugin now also queries the
`data_retention` and includes it in the EBT document.

Example document returned by `GET _data_stream/<ds
name>/?filter_path=data_streams.name,data_streams.indices,data_streams.lifecycle.enabled,data_streams.lifecycle.data_retention`

```json
{
  "data_streams": [
    {
      "name": "dsl-test",
      "indices": [
        {
          "index_name": ".ds-dsl-test-2025.12.08-000001",
          "index_uuid": "h9nu5fEIQJ-ObVemiXTPqg",
          "managed_by": "Data stream lifecycle",
          "prefer_ilm": true,
          "index_mode": "standard"
        },
        {
          "index_name": ".ds-dsl-test-2025.12.08-000002",
          "index_uuid": "VKp5OURcTIquxTmatmNz3g",
          "managed_by": "Data stream lifecycle",
          "prefer_ilm": true,
          "index_mode": "standard"
        }
      ],
      "lifecycle": {
        "enabled": true,
        "data_retention": "1h"
      }
    }
  ]
}
```

### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [ ] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
- [ ] Review the [backport
guidelines](https://docs.google.com/document/d/1VyN5k91e5OVumlc0Gb9RPa3h1ewuPE705nRtioPiTvY/edit?usp=sharing)
and apply applicable `backport:*` labels.

Author: szaffarano

x-pack/platform/plugins/private/indices_metadata/server/lib/ebt/events.ts

@@ -31,6 +31,19 @@ export const DATA_STREAM_EVENT: EventTypeOpts<DataStreams> = {
             type: 'keyword',
             _meta: { optional: true, description: 'ILM policy associated to the datastream' },
           },
+          dsl: {
+            properties: {
+              enabled: {
+                type: 'boolean',
+                _meta: { description: 'Whether the data stream is enabled' },
+              },
+              data_retention: {
+                type: 'text',
+                _meta: { optional: true, description: 'Data retention period' },
+              },
+            },
+            _meta: { optional: true, description: 'Data stream lifecycle settings' },
+          },
           template: {
             type: 'keyword',
             _meta: { optional: true, description: 'Template associated to the datastream' },

x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.test.ts

@@ -71,6 +71,10 @@ describe('Indices Metadata - IndicesMetadataService', () => {
   const mockDataStreams: DataStream[] = [
     {
       datastream_name: 'test-datastream',
+      dsl: {
+        enabled: true,
+        data_retention: '30d',
+      },
       indices: [{ index_name: 'test-index-1', ilm_policy: 'policy1' }],
     },
   ];
@@ -365,6 +369,60 @@ describe('Indices Metadata - IndicesMetadataService', () => {
       expect(receiver.getIlmsStats).toHaveBeenCalledWith(['test-index-1']);
     });
 
+    it('should publish datastreams with DSL through full metadata flow', async () => {
+      const datastreamsWithDsl: DataStream[] = [
+        {
+          datastream_name: 'logs-test',
+          dsl: {
+            enabled: true,
+            data_retention: '90d',
+          },
+          indices: [
+            {
+              index_name: '.ds-logs-test-000001',
+              ilm_policy: 'logs-policy',
+            },
+          ],
+        },
+      ];
+
+      receiver.getIndices.mockResolvedValue(mockIndexSettings);
+      receiver.getDataStreams.mockResolvedValue(datastreamsWithDsl);
+      receiver.getIndexTemplatesStats.mockResolvedValue(mockIndexTemplates);
+      receiver.getIndicesStats.mockImplementation(async function* () {
+        yield* mockIndexStats;
+      });
+      receiver.isIlmStatsAvailable.mockResolvedValue(true);
+      receiver.getIlmsStats.mockImplementation(async function* () {
+        yield {
+          index_name: '.ds-logs-test-000001',
+          phase: 'hot',
+          age: '1d',
+          policy_name: 'logs-policy',
+        };
+      });
+      receiver.getIlmsPolicies.mockImplementation(async function* () {
+        yield { policy_name: 'logs-policy', modified_date: '2023-01-01', phases: {} };
+      });
+
+      await service['publishIndicesMetadata'](); // eslint-disable-line dot-notation
+
+      expect(sender.reportEBT).toHaveBeenCalledWith(
+        expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }),
+        {
+          items: expect.arrayContaining([
+            expect.objectContaining({
+              datastream_name: 'logs-test',
+              dsl: {
+                enabled: true,
+                data_retention: '90d',
+              },
+            }),
+          ]),
+        }
+      );
+    });
+
     it('should throw error when not initialized', async () => {
       const uninitializedService = new IndicesMetadataService(logger, configurationService);
 
@@ -441,6 +499,116 @@ describe('Indices Metadata - IndicesMetadataService', () => {
         expect(result).toBe(1);
         expect(logger.debug).toHaveBeenCalledWith('Data streams events sent', { count: 1 });
       });
+
+      it('should publish datastreams with DSL enabled and retention', () => {
+        const datastreamsWithDsl: DataStream[] = [
+          {
+            datastream_name: 'logs-app-prod',
+            dsl: {
+              enabled: true,
+              data_retention: '7d',
+            },
+            indices: [{ index_name: '.ds-logs-app-prod-000001' }],
+          },
+          {
+            datastream_name: 'metrics-system',
+            dsl: {
+              enabled: false,
+              data_retention: undefined,
+            },
+            indices: [{ index_name: '.ds-metrics-system-000001' }],
+          },
+        ];
+
+        const result = service['publishDatastreamsStats'](datastreamsWithDsl); // eslint-disable-line dot-notation
+
+        expect(sender.reportEBT).toHaveBeenCalledWith(
+          expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }),
+          {
+            items: [
+              expect.objectContaining({
+                datastream_name: 'logs-app-prod',
+                dsl: {
+                  enabled: true,
+                  data_retention: '7d',
+                },
+              }),
+              expect.objectContaining({
+                datastream_name: 'metrics-system',
+                dsl: {
+                  enabled: false,
+                  data_retention: undefined,
+                },
+              }),
+            ],
+          }
+        );
+        expect(result).toBe(2);
+      });
+
+      it('should handle datastreams without DSL field', () => {
+        const datastreamsWithoutDsl: DataStream[] = [
+          {
+            datastream_name: 'legacy-datastream',
+            indices: [{ index_name: '.ds-legacy-000001' }],
+          },
+        ];
+
+        const result = service['publishDatastreamsStats'](datastreamsWithoutDsl); // eslint-disable-line dot-notation
+
+        expect(sender.reportEBT).toHaveBeenCalledWith(
+          expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }),
+          {
+            items: [
+              {
+                datastream_name: 'legacy-datastream',
+                indices: [{ index_name: '.ds-legacy-000001' }],
+              },
+            ],
+          }
+        );
+        expect(result).toBe(1);
+      });
+
+      it('should publish mixed DSL configurations', () => {
+        const mixedDatastreams: DataStream[] = [
+          {
+            datastream_name: 'with-retention',
+            dsl: { enabled: true, data_retention: '365d' },
+            indices: [],
+          },
+          {
+            datastream_name: 'enabled-no-retention',
+            dsl: { enabled: true, data_retention: undefined },
+            indices: [],
+          },
+          {
+            datastream_name: 'disabled',
+            dsl: { enabled: false, data_retention: undefined },
+            indices: [],
+          },
+        ];
+
+        const result = service['publishDatastreamsStats'](mixedDatastreams); // eslint-disable-line dot-notation
+
+        expect(sender.reportEBT).toHaveBeenCalledWith(
+          expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }),
+          {
+            items: expect.arrayContaining([
+              expect.objectContaining({
+                dsl: { enabled: true, data_retention: '365d' },
+              }),
+              expect.objectContaining({
+                dsl: { enabled: true, data_retention: undefined },
+              }),
+              expect.objectContaining({
+                dsl: { enabled: false, data_retention: undefined },
+              }),
+            ]),
+          }
+        );
+        expect(result).toBe(3);
+      });
     });
 
     describe('publishIndicesSettings', () => {

x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.ts

@@ -257,6 +257,7 @@ export class IndicesMetadataService {
     this.logger.debug('Indices settings sent', { count: indicesSettings.items.length } as LogMeta);
     return indicesSettings.items.length;
   }
+
   private async publishIlmStats(indices: string[]): Promise<Set<string>> {
     const ilmNames = new Set<string>();
     const ilmsStats: IlmsStats = {

x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.types.ts

@@ -114,9 +114,16 @@ export interface Index {
 export interface DataStreams {
   items: DataStream[];
 }
+
+export interface DataStreamLifeCycle {
+  enabled: boolean;
+  data_retention?: string;
+}
+
 export interface DataStream {
   datastream_name: string;
   ilm_policy?: string;
+  dsl?: DataStreamLifeCycle;
   template?: string;
   indices?: Index[];
 }