diff --git a/packages/qualys_gav/changelog.yml b/packages/qualys_gav/changelog.yml index 8d9662d465e..7bf1ebf6972 100644 --- a/packages/qualys_gav/changelog.yml +++ b/packages/qualys_gav/changelog.yml @@ -1,4 +1,12 @@ # newer versions go on top +- version: "0.4.0" + changes: + - description: Map Qualys cloud fields to cloud ECS fields. + type: enhancement + link: https://github.com/elastic/integrations/pull/15810 + - description: Fix mapping for field qualys_gav.asset.inventory_list_data.inventory + type: bugfix + link: https://github.com/elastic/integrations/pull/15810 - version: "0.3.1" changes: - description: Added support for comma separated list of IPv4 addresses in network_interface list data. diff --git a/packages/qualys_gav/data_stream/asset/agent/stream/cel.yml.hbs b/packages/qualys_gav/data_stream/asset/agent/stream/cel.yml.hbs index 2d85d9988ce..841f3adfd92 100644 --- a/packages/qualys_gav/data_stream/asset/agent/stream/cel.yml.hbs +++ b/packages/qualys_gav/data_stream/asset/agent/stream/cel.yml.hbs @@ -125,6 +125,9 @@ tags: {{#each tags as |tag|}} - {{tag}} {{/each}} +{{#if cloud_data}} + - {{cloud_data}} +{{/if}} {{#contains "forwarded" tags}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml index 93b69bbfd7a..eb88d7b2ae1 100644 --- a/packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml +++ b/packages/qualys_gav/data_stream/asset/elasticsearch/ingest_pipeline/default.yml @@ -23,6 +23,30 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. + # Set up tag logic for choosing which cloud metadata to include. + - append: + field: tags + value: ["elastic_cloud_data", "provider_cloud_data"] + if: ctx.tags?.contains('both') == true + - script: + lang: painless + if: ctx.tags?.contains('both') == true + source: ctx.tags.remove(ctx.tags.indexOf('both')); + - script: + lang: painless + if: ctx.cloud == null && ctx.tags?.contains('elastic_cloud_data') == true + source: ctx.tags.remove(ctx.tags.indexOf('elastic_cloud_data')); + - set: + field: _conf.want_provider_cloud + value: true + if: ctx.tags?.contains('provider_cloud_data') == true + - remove: + description: Remove elastic agent-provided cloud metadata fields if not requested. + field: cloud + if: ctx.tags != null && !ctx.tags.contains('elastic_cloud_data') + ignore_missing: true + ignore_failure: true + # parse the event JSON - rename: field: message @@ -649,6 +673,34 @@ processors: - append: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + field: qualys_gav.asset.inventory_list_data.inventory.created + target_field: qualys_gav.asset.inventory_list_data.inventory.created + tag: date_qualys_gav_asset_inventory_list_data_inventory_created + formats: + - UNIX_MS + - ISO8601 + if: ctx.qualys_gav?.asset?.inventory_list_data?.inventory?.created != null && ctx.qualys_gav.asset.inventory_list_data.inventory.created != '' + on_failure: + - remove: + field: qualys_gav.asset.inventory.created + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + field: qualys_gav.asset.inventory_list_data.inventory.last_updated + target_field: qualys_gav.asset.inventory_list_data.inventory.last_updated + tag: date_qualys_gav_asset_inventory_last_updated + formats: + - UNIX_MS + - ISO8601 + if: ctx.qualys_gav?.asset?.inventory_list_data?.inventory?.last_updated != null && ctx.qualys_gav.asset.inventory_list_data.inventory.last_updated != '' + on_failure: + - remove: + field: qualys_gav.asset.inventory_list_data.inventory.last_updated + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: field: qualys_gav.asset.activity.last_scanned_date target_field: qualys_gav.asset.activity.last_scanned_date @@ -1094,6 +1146,8 @@ processors: target_field: cloud.provider tag: lowercase_cloud_provider_from_asset_provider ignore_missing: true + + # Convert AWS EC2 specific fields - convert: field: qualys_gav.asset.cloud_provider.aws.ec2.has_agent tag: convert_qualys_gav_asset_cloud_provider_aws_ec2_has_agent_to_boolean @@ -1145,11 +1199,185 @@ processors: - append: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + + # Map/Convert AWS EC2 specific fields to ECS fields + - set: + field: cloud.account.id + tag: set_cloud_account_id_from_asset_cloud_provider_aws_ec2_account_id + copy_from: qualys_gav.asset.cloud_provider.aws.ec2.account_id + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.availability_zone + tag: set_cloud_availability_zone_from_asset_cloud_provider_aws_ec2_availability_zone + copy_from: qualys_gav.asset.cloud_provider.aws.ec2.availability_zone + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.instance.id + tag: set_cloud_instance_id_from_asset_cloud_provider_aws_ec2_instance_id + copy_from: qualys_gav.asset.cloud_provider.aws.ec2.instance_id + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.instance.name + tag: set_cloud_instance_name_from_asset_cloud_provider_aws_ec2_hostname + copy_from: qualys_gav.asset.cloud_provider.aws.ec2.hostname + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.machine.type + tag: set_cloud_machine_type_from_asset_cloud_provider_aws_ec2_instance_type + copy_from: qualys_gav.asset.cloud_provider.aws.ec2.instance_type + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.region + tag: set_cloud_region_from_asset_cloud_provider_aws_ec2_region_code + copy_from: qualys_gav.asset.cloud_provider.aws.ec2.region.code + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.service.name + tag: set_cloud_service_name + if: ctx._conf?.want_provider_cloud == true && ctx.cloud?.provider == 'aws' && ctx.qualys_gav?.asset?.cloud_provider?.aws?.ec2 != null + value: ec2 + + # Map/Convert Azure specific fields to ECS fields + - set: + field: cloud.account.id + tag: set_cloud_account_id_from_asset_cloud_provider_azure_vm_subscription_id + copy_from: qualys_gav.asset.cloud_provider.azure.vm.subscription_id + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.instance.id + tag: set_cloud_instance_id_from_asset_cloud_provider_azure_vm_vm_id + copy_from: qualys_gav.asset.cloud_provider.azure.vm.vm_id + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.instance.name + tag: set_cloud_instance_name_from_asset_cloud_provider_azure_vm_name + copy_from: qualys_gav.asset.cloud_provider.azure.vm.name + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.machine.type + tag: set_cloud_machine_type_from_asset_cloud_provider_azure_vm_size + copy_from: qualys_gav.asset.cloud_provider.azure.vm.size + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.region + tag: set_cloud_region_from_asset_cloud_provider_azure_vm_location + copy_from: qualys_gav.asset.cloud_provider.azure.vm.location + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.service.name + tag: set_cloud_service_name + if: ctx._conf?.want_provider_cloud == true && ctx.cloud?.provider == 'azure' && ctx.qualys_gav?.asset?.cloud_provider?.azure?.vm != null + value: vm + + # Map/Convert GCP specific fields to ECS fields + - set: + field: cloud.account.id + tag: set_cloud_account_id_from_asset_cloud_provider_gcp_compute_project_id + copy_from: qualys_gav.asset.cloud_provider.gcp.compute.project_id + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.availability_zone + tag: set_cloud_availability_zone_from_asset_cloud_provider_gcp_compute_zone + copy_from: qualys_gav.asset.cloud_provider.gcp.compute.zone + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.instance.id + tag: set_cloud_instance_id_from_asset_cloud_provider_gcp_compute_instance_id + copy_from: qualys_gav.asset.cloud_provider.gcp.compute.instance_id + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.instance.name + tag: set_cloud_instance_name_from_asset_cloud_provider_gcp_compute_hostname + copy_from: qualys_gav.asset.cloud_provider.gcp.compute.hostname + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.machine.type + tag: set_cloud_machine_type_from_asset_cloud_provider_gcp_compute_machine_type + copy_from: qualys_gav.asset.cloud_provider.gcp.compute.machine_type + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.project.id + tag: set_cloud_project_id_from_asset_cloud_provider_gcp_compute_project_id + copy_from: qualys_gav.asset.cloud_provider.gcp.compute.project_id + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - grok: + field: qualys_gav.asset.cloud_provider.gcp.compute.zone + tag: grok_cloud_region_from_asset_cloud_provider_gcp_compute_zone + patterns: + - '%{GREEDYDATA:cloud.region}-%{WORD}$' + if: ctx._conf?.want_provider_cloud == true + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: cloud.service.name + tag: set_cloud_service_name + if: ctx._conf?.want_provider_cloud == true && ctx.cloud?.provider == 'gcp' && ctx.qualys_gav?.asset?.cloud_provider?.gcp?.compute != null + value: compute + + # Convert IBM specific fields + - convert: + field: qualys_gav.asset.cloud_provider.ibm.virtual_server.datacenter_id + tag: convert_qualys_gav_asset_cloud_provider_ibm_virtual_server_datacenter_id_to_string + type: string + ignore_missing: true - convert: field: qualys_gav.asset.cloud_provider.ibm.virtual_server.ibm_id tag: convert_qualys_gav_asset_cloud_provider_ibm_virtual_server_ibm_id_to_string type: string ignore_missing: true + + # Map/Convert IBM specific fields to ECS fields + - set: + field: cloud.account.id + tag: set_cloud_account_id_from_asset_cloud_provider_ibm_virtual_server_datacenter_id + copy_from: qualys_gav.asset.cloud_provider.ibm.virtual_server.datacenter_id + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.instance.id + tag: set_cloud_instance_id_from_asset_cloud_provider_ibm_virtual_server_ibm_id + copy_from: qualys_gav.asset.cloud_provider.ibm.virtual_server.ibm_id + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.instance.name + tag: set_cloud_instance_name_from_asset_cloud_provider_ibm_virtual_server_device_name + copy_from: qualys_gav.asset.cloud_provider.ibm.virtual_server.device_name + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.availability_zone + tag: set_cloud_availability_zone_from_asset_cloud_provider_ibm_virtual_server_location + copy_from: qualys_gav.asset.cloud_provider.ibm.virtual_server.location + if: ctx._conf?.want_provider_cloud == true + ignore_empty_value: true + - set: + field: cloud.service.name + tag: set_cloud_service_name + if: ctx._conf?.want_provider_cloud == true && ctx.cloud?.provider == 'azure' && ctx.qualys_gav?.asset?.cloud_provider?.ibm?.virtual_server != null + value: virtual_server + + # Map risk scores - convert: field: qualys_gav.asset.risk_score tag: convert_qualys_gav_asset_risk_score_to_float @@ -1822,6 +2050,11 @@ processors: tag: remove_custom_duplicate_fields ignore_missing: true if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') + - remove: + field: + - _conf + tag: remove_temporary_fields + ignore_missing: true # Cleanup - script: diff --git a/packages/qualys_gav/data_stream/asset/fields/fields.yml b/packages/qualys_gav/data_stream/asset/fields/fields.yml index cb026dcfcc0..c6d29fb4fdf 100644 --- a/packages/qualys_gav/data_stream/asset/fields/fields.yml +++ b/packages/qualys_gav/data_stream/asset/fields/fields.yml @@ -394,6 +394,18 @@ type: date - name: source type: keyword + - name: inventory_list_data + type: group + fields: + - name: inventory + type: group + fields: + - name: source + type: keyword + - name: created + type: date + - name: last_updated + type: date - name: is_container_host type: boolean - name: isp diff --git a/packages/qualys_gav/data_stream/asset/manifest.yml b/packages/qualys_gav/data_stream/asset/manifest.yml index da7fa05f4d5..19cade6c266 100644 --- a/packages/qualys_gav/data_stream/asset/manifest.yml +++ b/packages/qualys_gav/data_stream/asset/manifest.yml @@ -66,6 +66,23 @@ streams: required: true show_user: false default: 30s + - name: cloud_data + title: Cloud Metadata Source + description: What source to use to populate `cloud.*` fields. + type: select + multi: false + required: true + show_user: false + options: + - text: Elastic Only + value: elastic_cloud_data + - text: Provider Only + value: provider_cloud_data + - text: Elastic and Provider + value: both + - text: None + value: '' + default: both - name: preserve_duplicate_custom_fields required: false title: Preserve duplicate custom fields diff --git a/packages/qualys_gav/data_stream/asset/sample_event.json b/packages/qualys_gav/data_stream/asset/sample_event.json index e8b113ddda8..763cebb58a9 100644 --- a/packages/qualys_gav/data_stream/asset/sample_event.json +++ b/packages/qualys_gav/data_stream/asset/sample_event.json @@ -1,18 +1,33 @@ { - "@timestamp": "2025-09-26T13:07:03.724Z", + "@timestamp": "2025-10-31T13:45:09.114Z", "agent": { - "ephemeral_id": "3729163b-132c-437f-b8a8-d4200aef9b6b", - "id": "0ef8fd67-1afe-4495-99f1-bae4e45a64b7", - "name": "elastic-agent-83075", + "ephemeral_id": "610611d4-7d6e-4d92-875d-e36be3592a28", + "id": "543bc156-60b7-4837-8416-d9e6821a67d7", + "name": "elastic-agent-44463", "type": "filebeat", - "version": "9.1.4" + "version": "9.2.0" }, "cloud": { - "provider": "aws" + "account": { + "id": "1234" + }, + "availability_zone": "us-west-2a", + "instance": { + "id": "instanceId_value", + "name": "hostname_value" + }, + "machine": { + "type": "m4.large" + }, + "provider": "aws", + "region": "us-west-2", + "service": { + "name": "ec2" + } }, "data_stream": { "dataset": "qualys_gav.asset", - "namespace": "72894", + "namespace": "83683", "type": "logs" }, "device": { @@ -25,9 +40,9 @@ "version": "8.17.0" }, "elastic_agent": { - "id": "0ef8fd67-1afe-4495-99f1-bae4e45a64b7", + "id": "543bc156-60b7-4837-8416-d9e6821a67d7", "snapshot": false, - "version": "9.1.4" + "version": "9.2.0" }, "event": { "agent_id_status": "verified", @@ -36,7 +51,7 @@ ], "created": "2025-07-09T14:21:12.000Z", "dataset": "qualys_gav.asset", - "ingested": "2025-09-26T13:07:06Z", + "ingested": "2025-10-31T13:45:12Z", "kind": "event", "module": "qualys_gav", "risk_score": 0, @@ -510,7 +525,8 @@ "preserve_duplicate_custom_fields", "hide_sensitive", "forwarded", - "qualys_gav-asset" + "qualys_gav-asset", + "provider_cloud_data" ], "user": { "name": "test_user" diff --git a/packages/qualys_gav/docs/README.md b/packages/qualys_gav/docs/README.md index b29677d1817..9de67a43a24 100644 --- a/packages/qualys_gav/docs/README.md +++ b/packages/qualys_gav/docs/README.md @@ -57,20 +57,35 @@ An example event for `asset` looks as following: ```json { - "@timestamp": "2025-09-26T13:07:03.724Z", + "@timestamp": "2025-10-31T13:45:09.114Z", "agent": { - "ephemeral_id": "3729163b-132c-437f-b8a8-d4200aef9b6b", - "id": "0ef8fd67-1afe-4495-99f1-bae4e45a64b7", - "name": "elastic-agent-83075", + "ephemeral_id": "610611d4-7d6e-4d92-875d-e36be3592a28", + "id": "543bc156-60b7-4837-8416-d9e6821a67d7", + "name": "elastic-agent-44463", "type": "filebeat", - "version": "9.1.4" + "version": "9.2.0" }, "cloud": { - "provider": "aws" + "account": { + "id": "1234" + }, + "availability_zone": "us-west-2a", + "instance": { + "id": "instanceId_value", + "name": "hostname_value" + }, + "machine": { + "type": "m4.large" + }, + "provider": "aws", + "region": "us-west-2", + "service": { + "name": "ec2" + } }, "data_stream": { "dataset": "qualys_gav.asset", - "namespace": "72894", + "namespace": "83683", "type": "logs" }, "device": { @@ -83,9 +98,9 @@ An example event for `asset` looks as following: "version": "8.17.0" }, "elastic_agent": { - "id": "0ef8fd67-1afe-4495-99f1-bae4e45a64b7", + "id": "543bc156-60b7-4837-8416-d9e6821a67d7", "snapshot": false, - "version": "9.1.4" + "version": "9.2.0" }, "event": { "agent_id_status": "verified", @@ -94,7 +109,7 @@ An example event for `asset` looks as following: ], "created": "2025-07-09T14:21:12.000Z", "dataset": "qualys_gav.asset", - "ingested": "2025-09-26T13:07:06Z", + "ingested": "2025-10-31T13:45:12Z", "kind": "event", "module": "qualys_gav", "risk_score": 0, @@ -568,7 +583,8 @@ An example event for `asset` looks as following: "preserve_duplicate_custom_fields", "hide_sensitive", "forwarded", - "qualys_gav-asset" + "qualys_gav-asset", + "provider_cloud_data" ], "user": { "name": "test_user" @@ -742,6 +758,9 @@ An example event for `asset` looks as following: | qualys_gav.asset.inventory.created | | date | | qualys_gav.asset.inventory.last_updated | | date | | qualys_gav.asset.inventory.source | | keyword | +| qualys_gav.asset.inventory_list_data.inventory.created | | date | +| qualys_gav.asset.inventory_list_data.inventory.last_updated | | date | +| qualys_gav.asset.inventory_list_data.inventory.source | | keyword | | qualys_gav.asset.is_container_host | | boolean | | qualys_gav.asset.isp | | keyword | | qualys_gav.asset.last_boot | | date | diff --git a/packages/qualys_gav/elasticsearch/transform/latest_asset/fields/fields.yml b/packages/qualys_gav/elasticsearch/transform/latest_asset/fields/fields.yml index cb026dcfcc0..c6d29fb4fdf 100644 --- a/packages/qualys_gav/elasticsearch/transform/latest_asset/fields/fields.yml +++ b/packages/qualys_gav/elasticsearch/transform/latest_asset/fields/fields.yml @@ -394,6 +394,18 @@ type: date - name: source type: keyword + - name: inventory_list_data + type: group + fields: + - name: inventory + type: group + fields: + - name: source + type: keyword + - name: created + type: date + - name: last_updated + type: date - name: is_container_host type: boolean - name: isp diff --git a/packages/qualys_gav/manifest.yml b/packages/qualys_gav/manifest.yml index 8d4ca22efa5..80d759eae8e 100644 --- a/packages/qualys_gav/manifest.yml +++ b/packages/qualys_gav/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: qualys_gav title: Qualys Global AssetView -version: 0.3.1 +version: 0.4.0 description: Collect logs from Qualys Global AssetView with Elastic Agent. type: integration categories: