[9.1] Add datastream lifecycle support to indices metadata (#245548) (#245566)

kibanamachine · szaffarano · web-flow · commit 253000b12cc2 · 2025-12-08T21:38:18.000+01:00
# Backport This will backport the following commits from `main` to `9.1`: - [Add datastream lifecycle support to indices metadata (#245548)](#245548)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)  Co-authored-by: Sebastián Zaffarano <sebastian.zaffarano@elastic.co>
diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/ebt/events.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/ebt/events.ts
@@ -31,6 +31,19 @@ export const DATA_STREAM_EVENT: EventTypeOpts<DataStreams> = {
             type: 'keyword',
             _meta: { optional: true, description: 'ILM policy associated to the datastream' },
           },
+          dsl: {
+            properties: {
+              enabled: {
+                type: 'boolean',
+                _meta: { description: 'Whether the data stream is enabled' },
+              },
+              data_retention: {
+                type: 'text',
+                _meta: { optional: true, description: 'Data retention period' },
+              },
+            },
+            _meta: { optional: true, description: 'Data stream lifecycle settings' },
+          },
           template: {
             type: 'keyword',
             _meta: { optional: true, description: 'Template associated to the datastream' },
diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.test.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.test.ts
@@ -71,6 +71,10 @@ describe('Indices Metadata - IndicesMetadataService', () => {
   const mockDataStreams: DataStream[] = [
     {
       datastream_name: 'test-datastream',
+      dsl: {
+        enabled: true,
+        data_retention: '30d',
+      },
       indices: [{ index_name: 'test-index-1', ilm_policy: 'policy1' }],
     },
   ];
@@ -365,6 +369,60 @@ describe('Indices Metadata - IndicesMetadataService', () => {
       expect(receiver.getIlmsStats).toHaveBeenCalledWith(['test-index-1']);
     });
 
+    it('should publish datastreams with DSL through full metadata flow', async () => {
+      const datastreamsWithDsl: DataStream[] = [
+        {
+          datastream_name: 'logs-test',
+          dsl: {
+            enabled: true,
+            data_retention: '90d',
+          },
+          indices: [
+            {
+              index_name: '.ds-logs-test-000001',
+              ilm_policy: 'logs-policy',
+            },
+          ],
+        },
+      ];
+
+      receiver.getIndices.mockResolvedValue(mockIndexSettings);
+      receiver.getDataStreams.mockResolvedValue(datastreamsWithDsl);
+      receiver.getIndexTemplatesStats.mockResolvedValue(mockIndexTemplates);
+      receiver.getIndicesStats.mockImplementation(async function* () {
+        yield* mockIndexStats;
+      });
+      receiver.isIlmStatsAvailable.mockResolvedValue(true);
+      receiver.getIlmsStats.mockImplementation(async function* () {
+        yield {
+          index_name: '.ds-logs-test-000001',
+          phase: 'hot',
+          age: '1d',
+          policy_name: 'logs-policy',
+        };
+      });
+      receiver.getIlmsPolicies.mockImplementation(async function* () {
+        yield { policy_name: 'logs-policy', modified_date: '2023-01-01', phases: {} };
+      });
+
+      await service['publishIndicesMetadata'](); // eslint-disable-line dot-notation
+
+      expect(sender.reportEBT).toHaveBeenCalledWith(
+        expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }),
+        {
+          items: expect.arrayContaining([
+            expect.objectContaining({
+              datastream_name: 'logs-test',
+              dsl: {
+                enabled: true,
+                data_retention: '90d',
+              },
+            }),
+          ]),
+        }
+      );
+    });
+
     it('should throw error when not initialized', async () => {
       const uninitializedService = new IndicesMetadataService(logger, configurationService);
 
@@ -438,6 +496,116 @@ describe('Indices Metadata - IndicesMetadataService', () => {
         expect(result).toBe(1);
         expect(logger.debug).toHaveBeenCalledWith('Data streams events sent', { count: 1 });
       });
+
+      it('should publish datastreams with DSL enabled and retention', () => {
+        const datastreamsWithDsl: DataStream[] = [
+          {
+            datastream_name: 'logs-app-prod',
+            dsl: {
+              enabled: true,
+              data_retention: '7d',
+            },
+            indices: [{ index_name: '.ds-logs-app-prod-000001' }],
+          },
+          {
+            datastream_name: 'metrics-system',
+            dsl: {
+              enabled: false,
+              data_retention: undefined,
+            },
+            indices: [{ index_name: '.ds-metrics-system-000001' }],
+          },
+        ];
+
+        const result = service['publishDatastreamsStats'](datastreamsWithDsl); // eslint-disable-line dot-notation
+
+        expect(sender.reportEBT).toHaveBeenCalledWith(
+          expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }),
+          {
+            items: [
+              expect.objectContaining({
+                datastream_name: 'logs-app-prod',
+                dsl: {
+                  enabled: true,
+                  data_retention: '7d',
+                },
+              }),
+              expect.objectContaining({
+                datastream_name: 'metrics-system',
+                dsl: {
+                  enabled: false,
+                  data_retention: undefined,
+                },
+              }),
+            ],
+          }
+        );
+        expect(result).toBe(2);
+      });
+
+      it('should handle datastreams without DSL field', () => {
+        const datastreamsWithoutDsl: DataStream[] = [
+          {
+            datastream_name: 'legacy-datastream',
+            indices: [{ index_name: '.ds-legacy-000001' }],
+          },
+        ];
+
+        const result = service['publishDatastreamsStats'](datastreamsWithoutDsl); // eslint-disable-line dot-notation
+
+        expect(sender.reportEBT).toHaveBeenCalledWith(
+          expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }),
+          {
+            items: [
+              {
+                datastream_name: 'legacy-datastream',
+                indices: [{ index_name: '.ds-legacy-000001' }],
+              },
+            ],
+          }
+        );
+        expect(result).toBe(1);
+      });
+
+      it('should publish mixed DSL configurations', () => {
+        const mixedDatastreams: DataStream[] = [
+          {
+            datastream_name: 'with-retention',
+            dsl: { enabled: true, data_retention: '365d' },
+            indices: [],
+          },
+          {
+            datastream_name: 'enabled-no-retention',
+            dsl: { enabled: true, data_retention: undefined },
+            indices: [],
+          },
+          {
+            datastream_name: 'disabled',
+            dsl: { enabled: false, data_retention: undefined },
+            indices: [],
+          },
+        ];
+
+        const result = service['publishDatastreamsStats'](mixedDatastreams); // eslint-disable-line dot-notation
+
+        expect(sender.reportEBT).toHaveBeenCalledWith(
+          expect.objectContaining({ eventType: DATA_STREAM_EVENT.eventType }),
+          {
+            items: expect.arrayContaining([
+              expect.objectContaining({
+                dsl: { enabled: true, data_retention: '365d' },
+              }),
+              expect.objectContaining({
+                dsl: { enabled: true, data_retention: undefined },
+              }),
+              expect.objectContaining({
+                dsl: { enabled: false, data_retention: undefined },
+              }),
+            ]),
+          }
+        );
+        expect(result).toBe(3);
+      });
     });
 
     describe('publishIndicesSettings', () => {
diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.ts
@@ -257,6 +257,7 @@ export class IndicesMetadataService {
     this.logger.debug('Indices settings sent', { count: indicesSettings.items.length } as LogMeta);
     return indicesSettings.items.length;
   }
+
   private async publishIlmStats(indices: string[]): Promise<Set<string>> {
     const ilmNames = new Set<string>();
     const ilmsStats: IlmsStats = {
diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.types.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/indices_metadata.types.ts
@@ -113,9 +113,16 @@ export interface Index {
 export interface DataStreams {
   items: DataStream[];
 }
+
+export interface DataStreamLifeCycle {
+  enabled: boolean;
+  data_retention?: string;
+}
+
 export interface DataStream {
   datastream_name: string;
   ilm_policy?: string;
+  dsl?: DataStreamLifeCycle;
   template?: string;
   indices?: Index[];
 }
diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.test.ts
diff --git a/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts b/x-pack/platform/plugins/private/indices_metadata/server/lib/services/receiver.ts

Original file line number	Diff line number	Diff line change
`@@ -257,6 +257,7 @@ export class IndicesMetadataService {`
`257`	`257`	`this.logger.debug('Indices settings sent', { count: indicesSettings.items.length } as LogMeta);`
`258`	`258`	`return indicesSettings.items.length;`
`259`	`259`	`}`
	`260`	`+`
`260`	`261`	`private async publishIlmStats(indices: string[]): Promise<Set<string>> {`
`261`	`262`	`const ilmNames = new Set<string>();`
`262`	`263`	`const ilmsStats: IlmsStats = {`