Skip to content

Commit 3678696

Browse files
zmoogzmoog
authored
[Azure Logs] Expand date formats for parsing time fields (#16328)
We are revising the date processor format list to improve robustness. We've prioritized the formats based on usage frequency: the first format is the most common, followed by necessary fallbacks. This addresses the fact that Azure Logs don't adhere to a single format. This approach maximizes efficiency while maintaining parsing resilience.
1 parent 007c106 commit 3678696

File tree

41 files changed

+3046
-37
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+3046
-37
lines changed

‎packages/azure/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
- version: "1.31.1"
2+
changes:
3+
- description: "Expand date formats for parsing time fields: ISO8601, `M/d/yyyy h:mm:ss a XXX`, `M/d/yyyy h:mm:ss a`, `M/d/yyyy H:mm:ss`, `yyyy-MM-ddTH:mm:ss.SSSZ`."
4+
type: bugfix
5+
link: https://github.com/elastic/integrations/pull/16328
16
- version: "1.31.0"
27
changes:
38
- description: |

‎packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-edgecases.log-expected.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -550,4 +550,4 @@
550550
]
551551
}
552552
]
553-
}
553+
}

‎packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-identity.log-expected.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -270,4 +270,4 @@
270270
]
271271
}
272272
]
273-
}
273+
}

‎packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-time-formats-corner-cases.log‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{"time": "01/09/2007 09:41:00", "category": "Administrative"}
2+
{"time": "1/9/2007 09:41:00", "category": "Administrative"}
3+
{"time": "01/09/2007 09:41:00 AM", "category": "Administrative"}
4+
{"time": "1/9/2007 9:41:00 AM", "category": "Administrative"}
5+
{"time": "1/9/2007 10:41:00 AM +01:00", "category": "Administrative"}
6+
{"time": "2007-01-09T09:41:00", "category": "Administrative"}
7+
{"time": "2007-01-09T09:41:00.22Z", "category": "Administrative"}
8+
{"time": "2007-01-09T09:41:00.6816663Z", "category": "Administrative"}
9+
{"time": "2007-01-09T09:41:00.535404056Z", "category": "Administrative"}
10+
{"time": "2007-01-09T09:41:00.992099+00:00", "category": "Administrative"}
11+
{"time": "2007-01-09T11:41:00+02:00", "category": "Administrative"}

‎packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-time-formats-corner-cases.log-expected.json‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
{
2+
"expected": [
3+
{
4+
"@timestamp": "2007-01-09T09:41:00.000Z",
5+
"azure": {
6+
"activitylogs": {
7+
"category": "Administrative",
8+
"event_category": "Administrative"
9+
}
10+
},
11+
"cloud": {
12+
"provider": "azure"
13+
},
14+
"ecs": {
15+
"version": "8.11.0"
16+
},
17+
"event": {
18+
"kind": "event",
19+
"original": "{\"time\": \"01/09/2007 09:41:00\", \"category\": \"Administrative\"}"
20+
},
21+
"tags": [
22+
"preserve_original_event"
23+
]
24+
},
25+
{
26+
"@timestamp": "2007-01-09T09:41:00.000Z",
27+
"azure": {
28+
"activitylogs": {
29+
"category": "Administrative",
30+
"event_category": "Administrative"
31+
}
32+
},
33+
"cloud": {
34+
"provider": "azure"
35+
},
36+
"ecs": {
37+
"version": "8.11.0"
38+
},
39+
"event": {
40+
"kind": "event",
41+
"original": "{\"time\": \"1/9/2007 09:41:00\", \"category\": \"Administrative\"}"
42+
},
43+
"tags": [
44+
"preserve_original_event"
45+
]
46+
},
47+
{
48+
"@timestamp": "2007-01-09T09:41:00.000Z",
49+
"azure": {
50+
"activitylogs": {
51+
"category": "Administrative",
52+
"event_category": "Administrative"
53+
}
54+
},
55+
"cloud": {
56+
"provider": "azure"
57+
},
58+
"ecs": {
59+
"version": "8.11.0"
60+
},
61+
"event": {
62+
"kind": "event",
63+
"original": "{\"time\": \"01/09/2007 09:41:00 AM\", \"category\": \"Administrative\"}"
64+
},
65+
"tags": [
66+
"preserve_original_event"
67+
]
68+
},
69+
{
70+
"@timestamp": "2007-01-09T09:41:00.000Z",
71+
"azure": {
72+
"activitylogs": {
73+
"category": "Administrative",
74+
"event_category": "Administrative"
75+
}
76+
},
77+
"cloud": {
78+
"provider": "azure"
79+
},
80+
"ecs": {
81+
"version": "8.11.0"
82+
},
83+
"event": {
84+
"kind": "event",
85+
"original": "{\"time\": \"1/9/2007 9:41:00 AM\", \"category\": \"Administrative\"}"
86+
},
87+
"tags": [
88+
"preserve_original_event"
89+
]
90+
},
91+
{
92+
"@timestamp": "2007-01-09T09:41:00.000Z",
93+
"azure": {
94+
"activitylogs": {
95+
"category": "Administrative",
96+
"event_category": "Administrative"
97+
}
98+
},
99+
"cloud": {
100+
"provider": "azure"
101+
},
102+
"ecs": {
103+
"version": "8.11.0"
104+
},
105+
"event": {
106+
"kind": "event",
107+
"original": "{\"time\": \"1/9/2007 10:41:00 AM +01:00\", \"category\": \"Administrative\"}"
108+
},
109+
"tags": [
110+
"preserve_original_event"
111+
]
112+
},
113+
{
114+
"@timestamp": "2007-01-09T09:41:00.000Z",
115+
"azure": {
116+
"activitylogs": {
117+
"category": "Administrative",
118+
"event_category": "Administrative"
119+
}
120+
},
121+
"cloud": {
122+
"provider": "azure"
123+
},
124+
"ecs": {
125+
"version": "8.11.0"
126+
},
127+
"event": {
128+
"kind": "event",
129+
"original": "{\"time\": \"2007-01-09T09:41:00\", \"category\": \"Administrative\"}"
130+
},
131+
"tags": [
132+
"preserve_original_event"
133+
]
134+
},
135+
{
136+
"@timestamp": "2007-01-09T09:41:00.220Z",
137+
"azure": {
138+
"activitylogs": {
139+
"category": "Administrative",
140+
"event_category": "Administrative"
141+
}
142+
},
143+
"cloud": {
144+
"provider": "azure"
145+
},
146+
"ecs": {
147+
"version": "8.11.0"
148+
},
149+
"event": {
150+
"kind": "event",
151+
"original": "{\"time\": \"2007-01-09T09:41:00.22Z\", \"category\": \"Administrative\"}"
152+
},
153+
"tags": [
154+
"preserve_original_event"
155+
]
156+
},
157+
{
158+
"@timestamp": "2007-01-09T09:41:00.681Z",
159+
"azure": {
160+
"activitylogs": {
161+
"category": "Administrative",
162+
"event_category": "Administrative"
163+
}
164+
},
165+
"cloud": {
166+
"provider": "azure"
167+
},
168+
"ecs": {
169+
"version": "8.11.0"
170+
},
171+
"event": {
172+
"kind": "event",
173+
"original": "{\"time\": \"2007-01-09T09:41:00.6816663Z\", \"category\": \"Administrative\"}"
174+
},
175+
"tags": [
176+
"preserve_original_event"
177+
]
178+
},
179+
{
180+
"@timestamp": "2007-01-09T09:41:00.535Z",
181+
"azure": {
182+
"activitylogs": {
183+
"category": "Administrative",
184+
"event_category": "Administrative"
185+
}
186+
},
187+
"cloud": {
188+
"provider": "azure"
189+
},
190+
"ecs": {
191+
"version": "8.11.0"
192+
},
193+
"event": {
194+
"kind": "event",
195+
"original": "{\"time\": \"2007-01-09T09:41:00.535404056Z\", \"category\": \"Administrative\"}"
196+
},
197+
"tags": [
198+
"preserve_original_event"
199+
]
200+
},
201+
{
202+
"@timestamp": "2007-01-09T09:41:00.992Z",
203+
"azure": {
204+
"activitylogs": {
205+
"category": "Administrative",
206+
"event_category": "Administrative"
207+
}
208+
},
209+
"cloud": {
210+
"provider": "azure"
211+
},
212+
"ecs": {
213+
"version": "8.11.0"
214+
},
215+
"event": {
216+
"kind": "event",
217+
"original": "{\"time\": \"2007-01-09T09:41:00.992099+00:00\", \"category\": \"Administrative\"}"
218+
},
219+
"tags": [
220+
"preserve_original_event"
221+
]
222+
},
223+
{
224+
"@timestamp": "2007-01-09T09:41:00.000Z",
225+
"azure": {
226+
"activitylogs": {
227+
"category": "Administrative",
228+
"event_category": "Administrative"
229+
}
230+
},
231+
"cloud": {
232+
"provider": "azure"
233+
},
234+
"ecs": {
235+
"version": "8.11.0"
236+
},
237+
"event": {
238+
"kind": "event",
239+
"original": "{\"time\": \"2007-01-09T11:41:00+02:00\", \"category\": \"Administrative\"}"
240+
},
241+
"tags": [
242+
"preserve_original_event"
243+
]
244+
}
245+
]
246+
}

‎packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,17 @@ processors:
3131
field: azure.activitylogs.time
3232
target_field: '@timestamp'
3333
ignore_failure: true
34-
formats:
34+
# Order is prioritized by observed frequency for this log
35+
# category.
36+
# We expect the first format to be used in 99% of cases.
37+
# However, we include the other formats seen in other
38+
# categories as a necessary fallback.
39+
formats:
3540
- ISO8601
41+
- "M/d/yyyy h:mm:ss a XXX"
42+
- "M/d/yyyy h:mm:ss a"
43+
- "M/d/yyyy H:mm:ss"
44+
- yyyy-MM-dd'T'H:mm:ss.SSS'Z'
3645
- remove:
3746
field: azure.activitylogs.time
3847
ignore_missing: true

‎packages/azure/data_stream/application_gateway/_dev/test/pipeline/test-time-formats-corner-cases.log‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{"time": "01/09/2007 09:41:00", "category": "ApplicationGatewayAccess"}
2+
{"time": "1/9/2007 09:41:00", "category": "ApplicationGatewayAccess"}
3+
{"time": "01/09/2007 09:41:00 AM", "category": "ApplicationGatewayAccess"}
4+
{"time": "1/9/2007 9:41:00 AM", "category": "ApplicationGatewayAccess"}
5+
{"time": "1/9/2007 10:41:00 AM +01:00", "category": "ApplicationGatewayAccess"}
6+
{"time": "2007-01-09T09:41:00", "category": "ApplicationGatewayAccess"}
7+
{"time": "2007-01-09T09:41:00.22Z", "category": "ApplicationGatewayAccess"}
8+
{"time": "2007-01-09T09:41:00.6816663Z", "category": "ApplicationGatewayAccess"}
9+
{"time": "2007-01-09T09:41:00.535404056Z", "category": "ApplicationGatewayAccess"}
10+
{"time": "2007-01-09T09:41:00.992099+00:00", "category": "ApplicationGatewayAccess"}
11+
{"time": "2007-01-09T11:41:00+02:00", "category": "ApplicationGatewayAccess"}

0 commit comments

Comments
 (0)