meging with main and fixing diffs

Signed-off-by: Andreas Gkizas <[email protected]>

Author: gizas

.buildkite/hooks/pre-command

@@ -73,7 +73,7 @@ if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
     export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
 fi
 
-if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" &&  "${BUILDKITE_STEP_KEY}" == "reference-target-branch" ]]; then
+if [[ ( "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ) &&  "${BUILDKITE_STEP_KEY}" == "reference-target-branch" ]]; then
     # Get the commit from target branch in the first step (reference-target-branch).
     # This step MUST be the first one and not run in parallel with any other step to ensure
     # that there is just one value for this variable
@@ -91,7 +91,7 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-publish" ]]; then
     fi
 fi
 
-if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" ]]; then
+if [[ "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations" ]]; then
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN

.buildkite/hooks/pre-exit

@@ -4,7 +4,7 @@ source .buildkite/scripts/common.sh
 
 set -euo pipefail
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack)$ ]]; then
     # FIXME: update condition depending on the pipeline steps triggered
     if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
         unset ELASTIC_PACKAGE_AWS_ACCESS_KEY

.buildkite/pipeline.yml

@@ -70,6 +70,9 @@ steps:
       image: "${LINUX_AGENT_IMAGE}"
       cpu: "8"
       memory: "4G"
+    if: |
+      build.env('BUILDKITE_PULL_REQUEST') != "false" &&
+      build.env('BUILDKITE_PIPELINE_SLUG') == "integrations"
 
   - label: ":sonarqube: Continuous Code Inspection"
     soft_fail: true # FIXME: Coverage is failing, remove this after solving the issue
@@ -83,6 +86,8 @@ steps:
     command: ".buildkite/scripts/run_sonar_scanner.sh"
     artifact_paths:
       - build/test-coverage/coverage_merged.xml
+    if: |
+      build.env('BUILDKITE_PIPELINE_SLUG') == "integrations"
 
   - label: ":junit: Junit annotate"
     plugins:
@@ -108,4 +113,5 @@ steps:
     # run this step when if it is triggered by the daily job
     if: |
       build.source == "trigger_job" &&
+      build.env('BUILDKITE_PIPELINE_SLUG') == "integrations" &&
       build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily"

.buildkite/pull-requests.json

@@ -16,6 +16,22 @@
       "skip_ci_on_only_changed": ["^.github/", "^docs/"],
       "always_require_ci_on_changed": []
     },
+    {
+      "enabled": true,
+      "pipelineSlug": "integrations-test-stack",
+      "allow_org_users": true,
+      "allowed_repo_permissions": ["admin", "write"],
+      "allowed_list": [],
+      "set_commit_status": true,
+      "build_on_commit": false,
+      "build_on_comment": true,
+      "trigger_comment_regex": "^/test stack (7|8|9)\\.\\d+\\.\\d+(-SNAPSHOT)?$",
+      "always_trigger_comment_regex": "^/test stack (7|8|9)\\.\\d+\\.\\d+(-SNAPSHOT)?$",
+      "skip_ci_labels": [],
+      "skip_target_branches": [],
+      "skip_ci_on_only_changed": [],
+      "always_require_ci_on_changed": []
+    },
     {
       "enabled": false,
       "pipelineSlug": "integrations-schedule-daily",

.buildkite/scripts/trigger_integrations_in_parallel.sh

@@ -29,6 +29,16 @@ to="$(get_to_changeset)"
 
 echo "[DEBUG] Checking with commits: from: '${from}' to: '${to}'"
 
+# This variable does not exist in builds triggered automatically
+GITHUB_PR_TRIGGER_COMMENT="${GITHUB_PR_TRIGGER_COMMENT:-""}"
+
+if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-test-stack" && "${GITHUB_PR_TRIGGER_COMMENT}" =~ ^/test\ stack ]]; then
+    echo "--- Stack version set from Github comment"
+    STACK_VERSION=$(echo "$GITHUB_PR_TRIGGER_COMMENT" | cut -d " " -f 3)
+    export STACK_VERSION
+    echo "Use Elastic stack version from Github comment: ${STACK_VERSION}"
+fi
+
 packages_to_test=0
 
 for package in ${PACKAGE_LIST}; do

.github/CODEOWNERS

@@ -460,15 +460,19 @@
 /packages/cisco_meraki_metrics @elastic/obs-infraobs-integrations
 /packages/panw_metrics @elastic/obs-infraobs-integrations
 /packages/o365_metrics @elastic/obs-infraobs-integrations @elastic/security-service-integrations
-/packages/o365_metrics/data_stream/active_users @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/active_users_services_user_counts @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/groups_activity_group_detail @elastic/security-service-integrations
-/packages/o365_metrics/data_stream/mailbox_usage_detail @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/mailbox_usage_quota_status @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/onedrive_usage @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/onedrive_usage_account_counts @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/onedrive_usage_file_counts @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/onedrive_usage_storage @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/onedrive_usage_account_detail @elastic/security-service-integrations
 /packages/o365_metrics/data_stream/outlook_activity @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/outlook_app_usage @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/sharepoint_site_usage_storage @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/sharepoint_site_usage_detail @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/mailbox_usage_detail @elastic/obs-infraobs-integrations
+/packages/o365_metrics/data_stream/mailbox_usage_quota_status @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/outlook_app_usage_version_counts @elastic/obs-infraobs-integrations
-/packages/o365_metrics/data_stream/sharepoint_site_usage @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/teams_device_usage_user_counts @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/teams_user_activity_user_counts @elastic/obs-infraobs-integrations
 /packages/o365_metrics/data_stream/teams_user_activity_user_detail @elastic/security-service-integrations

.github/ISSUE_TEMPLATE/integration_bug.yml

@@ -53,6 +53,7 @@ body:
         - BBOT (Bighuge BLS OSINT Tool) [bbot]
         - Network Beaconing Identification [beaconing]
         - Beat [beat]
+        - BeyondInsight and Password Safe [beyondinsight_password_safe]
         - BitDefender [bitdefender]
         - Bitwarden [bitwarden]
         - blacklens.io [blacklens]

.github/ISSUE_TEMPLATE/integration_feature_request.yml

@@ -53,6 +53,7 @@ body:
         - BBOT (Bighuge BLS OSINT Tool) [bbot]
         - Network Beaconing Identification [beaconing]
         - Beat [beat]
+        - BeyondInsight and Password Safe [beyondinsight_password_safe]
         - BitDefender [bitdefender]
         - Bitwarden [bitwarden]
         - blacklens.io [blacklens]

catalog-info.yaml

@@ -64,6 +64,52 @@ spec:
         everyone:
           access_level: READ_ONLY
 
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-integrations-test-stack
+  description: 'Pipeline to test packages with a given Elastic stack version'
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/integrations-test-stack
+
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: platform-ingest
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: integrations-test-stack
+      description: 'Pipeline to test packages with a given Elastic stack version'
+    spec:
+      branch_configuration: "main backport-*"
+      pipeline_file: ".buildkite/pipeline.yml"
+      provider_settings:
+        build_pull_request_forks: false
+        build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot
+        publish_commit_status: true
+        build_tags: false
+        build_branches: false
+        filter_enabled: true
+        filter_condition: >-
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
+      repository: elastic/integrations
+      cancel_intermediate_builds: true
+      cancel_intermediate_builds_branch_filter: '!main !backport-*'
+      skip_intermediate_builds: true
+      skip_intermediate_builds_branch_filter: '!main !backport-*'
+      env:
+        ELASTIC_PR_COMMENTS_ENABLED: 'true'
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1

dev/testsreporter/_static/description.tmpl

@@ -1,11 +1,11 @@
 {{ .summary }}
-{{ if ne .failure "" -}}
+{{ if and (ne .failure nil) (ne .failure "") -}}
 Failure:
 ```
 {{ .failure }}
 ```
 {{- end }}
-{{- if ne .error "" -}}
+{{- if and (ne .error nil) (ne .error "") -}}
 Error:
 ```
 {{ .error }}