Skip to content

Commit b90febd

Browse files
flexitrevflexitrev
authored
Cloudflare Logpush - fix SQS worker setting (#13346)
1 parent 1f9154c commit b90febd

File tree

37 files changed

+144
-103
lines changed

37 files changed

+144
-103
lines changed

‎packages/cloudflare_logpush/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "1.35.2"
3+
changes:
4+
- description: Fix handling of SQS worker count configuration.
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/13346
27
- version: "1.35.1"
38
changes:
49
- description: Fix handling of http_request events missing `ClientRequestProtocol`.

‎packages/cloudflare_logpush/data_stream/access_request/agent/stream/aws-s3.yml.hbs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,6 @@ When using an S3 bucket, you can specify only one of the following options:
99

1010
{{#if collect_s3_logs}}
1111
{{! shared S3 bucket polling options }}
12-
{{#if number_of_workers }}
13-
number_of_workers: {{ number_of_workers }}
14-
{{/if}}
1512

1613
{{#if bucket_list_prefix }}
1714
bucket_list_prefix: {{ bucket_list_prefix }}
@@ -90,6 +87,11 @@ file_selectors:
9087

9188
{{/if}}
9289

90+
{{! allows number of workers to be configured for SQS queue and S3 buckets}}
91+
{{#if number_of_workers }}
92+
number_of_workers: {{ number_of_workers }}
93+
{{/if}}
94+
9395
{{#if access_key_id}}
9496
access_key_id: {{access_key_id}}
9597
{{/if}}

‎packages/cloudflare_logpush/data_stream/access_request/manifest.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,12 @@ streams:
115115
description: Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.
116116
- name: number_of_workers
117117
type: integer
118-
title: '[S3] Number of Workers'
118+
title: '[S3/SQS] Number of Workers'
119119
multi: false
120120
required: false
121121
show_user: true
122122
default: 5
123-
description: Number of workers that will process the S3 objects listed.
123+
description: Number of workers that will process the S3/SQS objects listed.
124124
- name: start_timestamp
125125
type: text
126126
title: "[S3] Start Timestamp"
@@ -157,7 +157,7 @@ streams:
157157
required: false
158158
show_user: true
159159
default: 5
160-
description: The maximum number of SQS messages that can be inflight at any time.
160+
description: Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.
161161
- name: file_selectors
162162
type: yaml
163163
title: '[SQS] File Selectors'

‎packages/cloudflare_logpush/data_stream/audit/agent/stream/aws-s3.yml.hbs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,6 @@ When using an S3 bucket, you can specify only one of the following options:
99

1010
{{#if collect_s3_logs}}
1111
{{! shared S3 bucket polling options }}
12-
{{#if number_of_workers }}
13-
number_of_workers: {{ number_of_workers }}
14-
{{/if}}
1512

1613
{{#if bucket_list_prefix }}
1714
bucket_list_prefix: {{ bucket_list_prefix }}
@@ -90,6 +87,11 @@ file_selectors:
9087

9188
{{/if}}
9289

90+
{{! allows number of workers to be configured for SQS queue and S3 buckets}}
91+
{{#if number_of_workers }}
92+
number_of_workers: {{ number_of_workers }}
93+
{{/if}}
94+
9395
{{#if access_key_id}}
9496
access_key_id: {{access_key_id}}
9597
{{/if}}

‎packages/cloudflare_logpush/data_stream/audit/manifest.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,12 @@ streams:
115115
description: Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.
116116
- name: number_of_workers
117117
type: integer
118-
title: '[S3] Number of Workers'
118+
title: '[S3/SQS] Number of Workers'
119119
multi: false
120120
required: false
121121
show_user: true
122122
default: 5
123-
description: Number of workers that will process the S3 objects listed.
123+
description: Number of workers that will process the S3/SQS objects listed.
124124
- name: start_timestamp
125125
type: text
126126
title: "[S3] Start Timestamp"
@@ -157,7 +157,7 @@ streams:
157157
required: false
158158
show_user: true
159159
default: 5
160-
description: The maximum number of SQS messages that can be inflight at any time.
160+
description: Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.
161161
- name: file_selectors
162162
type: yaml
163163
title: '[SQS] File Selectors'

‎packages/cloudflare_logpush/data_stream/casb/agent/stream/aws-s3.yml.hbs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,6 @@ When using an S3 bucket, you can specify only one of the following options:
99

1010
{{#if collect_s3_logs}}
1111
{{! shared S3 bucket polling options }}
12-
{{#if number_of_workers }}
13-
number_of_workers: {{ number_of_workers }}
14-
{{/if}}
1512

1613
{{#if bucket_list_prefix }}
1714
bucket_list_prefix: {{ bucket_list_prefix }}
@@ -90,6 +87,11 @@ file_selectors:
9087

9188
{{/if}}
9289

90+
{{! allows number of workers to be configured for SQS queue and S3 buckets}}
91+
{{#if number_of_workers }}
92+
number_of_workers: {{ number_of_workers }}
93+
{{/if}}
94+
9395
{{#if access_key_id}}
9496
access_key_id: {{access_key_id}}
9597
{{/if}}

‎packages/cloudflare_logpush/data_stream/casb/manifest.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,12 @@ streams:
115115
description: Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.
116116
- name: number_of_workers
117117
type: integer
118-
title: '[S3] Number of Workers'
118+
title: '[S3/SQS] Number of Workers'
119119
multi: false
120120
required: false
121121
show_user: true
122122
default: 5
123-
description: Number of workers that will process the S3 objects listed.
123+
description: Number of workers that will process the S3/SQS objects listed.
124124
- name: start_timestamp
125125
type: text
126126
title: "[S3] Start Timestamp"
@@ -157,7 +157,7 @@ streams:
157157
required: false
158158
show_user: true
159159
default: 5
160-
description: The maximum number of SQS messages that can be inflight at any time.
160+
description: Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.
161161
- name: file_selectors
162162
type: yaml
163163
title: '[SQS] File Selectors'

‎packages/cloudflare_logpush/data_stream/device_posture/agent/stream/aws-s3.yml.hbs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,6 @@ When using an S3 bucket, you can specify only one of the following options:
99

1010
{{#if collect_s3_logs}}
1111
{{! shared S3 bucket polling options }}
12-
{{#if number_of_workers }}
13-
number_of_workers: {{ number_of_workers }}
14-
{{/if}}
1512

1613
{{#if bucket_list_prefix }}
1714
bucket_list_prefix: {{ bucket_list_prefix }}
@@ -91,6 +88,11 @@ file_selectors:
9188

9289
{{/if}}
9390

91+
{{! allows number of workers to be configured for SQS queue and S3 buckets}}
92+
{{#if number_of_workers }}
93+
number_of_workers: {{ number_of_workers }}
94+
{{/if}}
95+
9496
{{#if access_key_id}}
9597
access_key_id: {{access_key_id}}
9698
{{/if}}

‎packages/cloudflare_logpush/data_stream/device_posture/manifest.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,12 @@ streams:
115115
description: Time interval for polling listing of the S3 bucket. Supported units for this parameter are h/m/s.
116116
- name: number_of_workers
117117
type: integer
118-
title: '[S3] Number of Workers'
118+
title: '[S3/SQS] Number of Workers'
119119
multi: false
120120
required: false
121121
show_user: true
122122
default: 5
123-
description: Number of workers that will process the S3 objects listed.
123+
description: Number of workers that will process the S3/SQS objects listed.
124124
- name: visibility_timeout
125125
type: text
126126
title: '[SQS] Visibility Timeout'
@@ -143,7 +143,7 @@ streams:
143143
required: false
144144
show_user: true
145145
default: 5
146-
description: The maximum number of SQS messages that can be inflight at any time.
146+
description: Deprecated in agent version 8.16.0, this parameter is ignored if present, use number_of_workers instead. The maximum number of SQS messages that can be inflight at any time.
147147
- name: file_selectors
148148
type: yaml
149149
title: '[SQS] File Selectors'

‎packages/cloudflare_logpush/data_stream/dns/agent/stream/aws-s3.yml.hbs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,6 @@ When using an S3 bucket, you can specify only one of the following options:
99

1010
{{#if collect_s3_logs}}
1111
{{! shared S3 bucket polling options }}
12-
{{#if number_of_workers }}
13-
number_of_workers: {{ number_of_workers }}
14-
{{/if}}
1512

1613
{{#if bucket_list_prefix }}
1714
bucket_list_prefix: {{ bucket_list_prefix }}
@@ -90,6 +87,11 @@ file_selectors:
9087

9188
{{/if}}
9289

90+
{{! allows number of workers to be configured for SQS queue and S3 buckets}}
91+
{{#if number_of_workers }}
92+
number_of_workers: {{ number_of_workers }}
93+
{{/if}}
94+
9395
{{#if access_key_id}}
9496
access_key_id: {{access_key_id}}
9597
{{/if}}

0 commit comments

Comments
 (0)