From 0d52fde3688bd68e3537ee05a0982c2bd91a1418 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 3 Mar 2025 10:19:57 +0000 Subject: [PATCH 1/3] Add space to entity store and entity-resolution-demo commands --- src/commands/asset_criticality.ts | 14 ++- src/commands/documents.ts | 1 - src/commands/entity-store.ts | 35 +++++-- src/commands/entity_resolution.ts | 19 ++-- src/constants.ts | 33 +++--- src/index.ts | 13 ++- src/utils/initialize_space.ts | 2 +- src/utils/kibana_api.ts | 164 +++++++++++++++++------------- 8 files changed, 172 insertions(+), 109 deletions(-) diff --git a/src/commands/asset_criticality.ts b/src/commands/asset_criticality.ts index a1c44fe..ef07a00 100644 --- a/src/commands/asset_criticality.ts +++ b/src/commands/asset_criticality.ts @@ -13,10 +13,12 @@ export const generateAssetCriticality = async ({ users, hosts, seed = generateNewSeed(), + space, }: { users: number; hosts: number; seed?: number; + space: string; }) => { faker.seed(seed); @@ -29,9 +31,17 @@ export const generateAssetCriticality = async ({ count: hosts, }); - await assignAssetCriticalityToEntities(generatedUsers, 'user.name'); + await assignAssetCriticalityToEntities({ + entities: generatedUsers, + field: 'user.name', + space, + }); console.log(`Assigned asset criticality to ${generatedUsers.length} users`); - await assignAssetCriticalityToEntities(generatedHosts, 'host.name'); + await assignAssetCriticalityToEntities({ + entities: generatedHosts, + field: 'host.name', + space, + }); console.log(`Assigned asset criticality to ${generatedHosts.length} hosts`); console.log('Finished generating asset criticality'); diff --git a/src/commands/documents.ts b/src/commands/documents.ts index e086f5e..b2ee99e 100644 --- a/src/commands/documents.ts +++ b/src/commands/documents.ts @@ -196,7 +196,6 @@ export const generateEvents = async (n: number) => { }; export const generateGraph = async ({ users = 100, maxHosts = 3 }) => { - //await alertIndexCheck(); TODO console.log('Generating alerts graph...'); type AlertOverride = { host: { name: string }; user: { name: string } }; diff --git a/src/commands/entity-store.ts b/src/commands/entity-store.ts index 3a060c0..ec58ea4 100644 --- a/src/commands/entity-store.ts +++ b/src/commands/entity-store.ts @@ -15,6 +15,7 @@ import { MappingTypeMapping, } from '@elastic/elasticsearch/lib/api/types'; import { getConfig } from '../get_config'; +import { initializeSpace } from '../utils'; const config = getConfig(); const client = getEsClient(); @@ -263,10 +264,12 @@ export const generateEvents = ( }, acc); }; -export const assignAssetCriticalityToEntities = async ( - entities: BaseEntity[], - field: string, -) => { +export const assignAssetCriticalityToEntities = async (opts: { + entities: BaseEntity[]; + field: string; + space?: string; +}) => { + const { entities, field, space } = opts; const chunks = chunk(entities, 10000); for (const chunk of chunks) { const records = chunk @@ -278,7 +281,7 @@ export const assignAssetCriticalityToEntities = async ( })); if (records.length > 0) { - await assignAssetCriticality(records); + await assignAssetCriticality(records, space); } } }; @@ -292,12 +295,14 @@ export const generateEntityStore = async ({ hosts = 10, services = 10, seed = generateNewSeed(), + space, options, }: { users: number; hosts: number; services: number; seed: number; + space?: string; options: string[]; }) => { if (options.includes(ENTITY_STORE_OPTIONS.seed)) { @@ -342,20 +347,32 @@ export const generateEntityStore = async ({ await ingestEvents(eventsForServices); console.log('Services events ingested'); + if (space && space !== 'default') { + await initializeSpace(space); + } + if (options.includes(ENTITY_STORE_OPTIONS.criticality)) { - await assignAssetCriticalityToEntities(generatedUsers, 'user.name'); + await assignAssetCriticalityToEntities({ + entities: generatedUsers, + field: 'user.name', + space, + }); console.log('Assigned asset criticality to users'); - await assignAssetCriticalityToEntities(generatedHosts, 'host.name'); + await assignAssetCriticalityToEntities({ + entities: generatedHosts, + field: 'host.name', + space, + }); console.log('Assigned asset criticality to hosts'); } if (options.includes(ENTITY_STORE_OPTIONS.riskEngine)) { - await enableRiskScore(); + await enableRiskScore(space); console.log('Risk score enabled'); } if (options.includes(ENTITY_STORE_OPTIONS.rule)) { - await createRule(); + await createRule({ space }); console.log('Rule created'); } diff --git a/src/commands/entity_resolution.ts b/src/commands/entity_resolution.ts index 25362a3..52340de 100644 --- a/src/commands/entity_resolution.ts +++ b/src/commands/entity_resolution.ts @@ -4,7 +4,7 @@ import { createRule, getRule, createComponentTemplate, - appendPathToKibanaNode, + buildKibanaUrl, } from '../utils/kibana_api'; import pMap from 'p-map'; import cliProgress from 'cli-progress'; @@ -213,7 +213,7 @@ const PACKAGES_TO_INSTALL = [ 'entityanalytics_entra_id', ]; -const installPackages = async () => { +const installPackages = async (space: string) => { console.log('Installing packages...'); const progress = new cliProgress.SingleBar( { @@ -225,7 +225,7 @@ const installPackages = async () => { await pMap( PACKAGES_TO_INSTALL, async (packageName) => { - await installPackage({ packageName }); + await installPackage({ packageName, space }); progress.increment(); }, { concurrency: 1 }, @@ -415,8 +415,8 @@ const importFile = async ( await batchIndexDocsWithProgress(batchGenerator, lineCountInFile); }; -const createMatchAllRule = async () => { - const rule = await getRule(RULE_ID); +const createMatchAllRule = async (space: string) => { + const rule = await getRule(RULE_ID, space); if (rule) { console.log('Match all rule already exists.'); @@ -425,6 +425,7 @@ const createMatchAllRule = async () => { await createRule({ id: RULE_ID, + space, }); console.log('Match all rule created.'); }; @@ -462,10 +463,12 @@ export const setupEntityResolutionDemo = async ({ mini = false, deleteData = false, keepEmails = false, + space, }: { mini: boolean; deleteData: boolean; keepEmails: boolean; + space: string; }) => { if (deleteData) { console.log('Deleting existing demo data first...'); @@ -474,9 +477,9 @@ export const setupEntityResolutionDemo = async ({ console.log(`Setting up${mini ? ' mini' : ''} entity resolution demo...`); // create a rule which matches everything, handy for exploring all the different entity views - await createMatchAllRule(); + await createMatchAllRule(space); // install the packages to get the mappings in place - await installPackages(); + await installPackages(space); // create @custom component templates to get user.name and user.email field mappings // which the inttegrations don't provide // we will eventually have to release a new version of the integrations to include these mappings @@ -493,7 +496,7 @@ Entity resolution demo setup complete. Now go and install the model! - CLICK HERE ---->> ${appendPathToKibanaNode('/app/security/entity_analytics_management')} <<---- CLICK HERE + CLICK HERE ---->> ${buildKibanaUrl({ path: '/app/security/entity_analytics_management', space })} <<---- CLICK HERE Once installed, ${mini ? 'Mark Hopkin should have matches' : 'See here:\n\n https://github.com/elastic/security-ml/blob/gus/entity_resoluton_data_generation/projects/entity_resolution_poc_2024/test_data_generation/seed_data_with_name_variations_and_user_agent_gen_and_groups.json \n\nfor all the seed data names'} `); diff --git a/src/constants.ts b/src/constants.ts index 5614c55..9852406 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -10,6 +10,15 @@ export const generateNewSeed = () => { return Math.round(Math.random() * 100000); }; +export const API_VERSIONS = { + public: { + v1: '2023-10-31', + }, + internal: { + v1: '1', + }, +}; + // API Endpoint URL's for Kibana export const RISK_SCORE_URL = '/internal/risk_score'; export const RISK_SCORE_DASHBOARD_URL = (entityType: 'host' | 'user') => @@ -17,22 +26,16 @@ export const RISK_SCORE_DASHBOARD_URL = (entityType: 'host' | 'user') => export const RISK_SCORE_SCORES_URL = '/internal/risk_score/scores'; export const RISK_SCORE_ENGINE_INIT_URL = '/internal/risk_score/engine/init'; export const ASSET_CRITICALITY_URL = '/api/asset_criticality'; -export const DETECTION_ENGINE_RULES_URL = (space?: string) => - space - ? `/s/${space}/api/detection_engine/rules` - : '/api/detection_engine/rules'; -export const COMPONENT_TEMPLATES_URL = (space?: string) => - space - ? `/s/${space}/api/index_management/component_templates` - : '/api/index_management/component_templates'; +export const ASSET_CRITICALITY_BULK_URL = '/api/asset_criticality/bulk'; +export const DETECTION_ENGINE_RULES_URL = '/api/detection_engine/rules'; +export const DETECTION_ENGINE_RULES_BULK_ACTION_URL = `${DETECTION_ENGINE_RULES_URL}/_bulk_action`; +export const COMPONENT_TEMPLATES_URL = + '/api/index_management/component_templates'; export const FLEET_EPM_PACKAGES_URL = ( packageName: string, version: string = 'latest', - space?: string, ) => { - let url = space - ? `/s/${space}/api/fleet/epm/packages/${packageName}` - : `/api/fleet/epm/packages/${packageName}`; + let url = `/api/fleet/epm/packages/${packageName}`; if (version !== 'latest') { url = `${url}/${version}`; } @@ -40,3 +43,9 @@ export const FLEET_EPM_PACKAGES_URL = ( }; export const SPACES_URL = '/api/spaces/space'; export const SPACE_URL = (space: string) => `/api/spaces/space/${space}`; + +export const ENTITY_ENGINES_URL = '/api/entity_store/engines'; +export const ENTITY_ENGINE_URL = (engineType: string) => + `${ENTITY_ENGINES_URL}/${engineType}`; +export const INIT_ENTITY_ENGINE_URL = (engineType: string) => + `${ENTITY_ENGINE_URL(engineType)}/init`; diff --git a/src/index.ts b/src/index.ts index b81b522..8f5f2b8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -156,15 +156,17 @@ program .option('--mini', 'Only load the mini dataset', false) .option('--delete', 'Delete old data', false) .option('--keep-emails', 'No Email variants', false) + .option('--space', 'space to use', 'default') .description('Load entity resolution demo data') - .action(({ mini, deleteData, keepEmails }) => { - setupEntityResolutionDemo({ mini, deleteData, keepEmails }); + .action(({ mini, deleteData, keepEmails, space }) => { + setupEntityResolutionDemo({ mini, deleteData, keepEmails, space }); }); program .command('entity-store') .description('Generate entity store') - .action(async () => { + .option('--space ', 'Space to create entity store in') + .action(async (options) => { const entityStoreAnswers = await checkbox< keyof typeof ENTITY_STORE_OPTIONS >({ @@ -226,6 +228,7 @@ program } generateEntityStore({ + space: options.space, users: parseIntBase10(userCount), hosts: parseIntBase10(hostCount), services: parseIntBase10(serviceCount), @@ -243,12 +246,14 @@ program .command('generate-asset-criticality') .option('-h ', 'number of hosts') .option('-u ', 'number of users') + .option('-s ', 'space') .description('Generate asset criticality for entities') .action(async (options) => { const users = parseInt(options.u || 10); const hosts = parseInt(options.h || 10); + const space = options.s || 'default'; - generateAssetCriticality({ users, hosts }); + generateAssetCriticality({ users, hosts, space }); }); program diff --git a/src/utils/initialize_space.ts b/src/utils/initialize_space.ts index 488a854..5351690 100644 --- a/src/utils/initialize_space.ts +++ b/src/utils/initialize_space.ts @@ -86,7 +86,7 @@ const waitForAlertIndexMapping = async ( const ensureSpaceExists = async (space: string) => { console.log(`Checking if space ${space} exists`); - if (await kibanaApi.getSpace(space)) { + if (await kibanaApi.doesSpaceExist(space)) { console.log(`Space ${space} exists`); return; } diff --git a/src/utils/kibana_api.ts b/src/utils/kibana_api.ts index 779ae7d..ae1bf4d 100644 --- a/src/utils/kibana_api.ts +++ b/src/utils/kibana_api.ts @@ -12,11 +12,21 @@ import { SPACE_URL, RISK_SCORE_URL, RISK_SCORE_DASHBOARD_URL, + ASSET_CRITICALITY_BULK_URL, + INIT_ENTITY_ENGINE_URL, + ENTITY_ENGINE_URL, + ENTITY_ENGINES_URL, + DETECTION_ENGINE_RULES_BULK_ACTION_URL, + API_VERSIONS, } from '../constants'; const config = getConfig(); -export const appendPathToKibanaNode = (path: string) => - urlJoin(config.kibana.node, path); + +export const buildKibanaUrl = (opts: { path: string; space?: string }) => { + const { path, space } = opts; + const pathWithSpace = space ? urlJoin(`/s/${space}`, path) : path; + return urlJoin(config.kibana.node, pathWithSpace); +}; type ResponseError = Error & { statusCode: number; responseData: unknown }; @@ -34,10 +44,14 @@ const throwResponseError = ( export const kibanaFetch = async ( path: string, params: object, - apiVersion = '1', - ignoreStatuses: number | number[] = [], + opts: { + ignoreStatuses?: number[] | number; + apiVersion?: string; + space?: string; + } = {}, ): Promise => { - const url = appendPathToKibanaNode(path); + const { ignoreStatuses, apiVersion = '1', space } = opts; + const url = buildKibanaUrl({ path, space }); const ignoreStatusesArray = Array.isArray(ignoreStatuses) ? ignoreStatuses : [ignoreStatuses]; @@ -62,7 +76,9 @@ export const kibanaFetch = async ( headers: headers, ...params, }); - const data = (await result.json()) as T; + const rawResponse = await result.text(); + // log response status + const data = rawResponse ? JSON.parse(rawResponse) : {}; if (!data || typeof data !== 'object') { throw new Error(); } @@ -77,18 +93,28 @@ export const kibanaFetch = async ( return data; }; -export const fetchRiskScore = async () => { - await kibanaFetch(RISK_SCORE_SCORES_URL, { - method: 'POST', - body: JSON.stringify({}), - }); +export const fetchRiskScore = async (space?: string) => { + await kibanaFetch( + RISK_SCORE_SCORES_URL, + { + method: 'POST', + body: JSON.stringify({}), + }, + { space }, + ); }; -export const enableRiskScore = async () => { - return kibanaFetch(RISK_SCORE_ENGINE_INIT_URL, { - method: 'POST', - body: JSON.stringify({}), - }); +export const enableRiskScore = async (space?: string) => { + return kibanaFetch( + RISK_SCORE_ENGINE_INIT_URL, + { + method: 'POST', + body: JSON.stringify({}), + }, + { + space, + }, + ); }; export const assignAssetCriticality = async ( @@ -97,15 +123,15 @@ export const assignAssetCriticality = async ( id_value: string; criticality_level: string; }>, - version: string = '2023-10-31', + space?: string, ) => { return kibanaFetch( - '/api/asset_criticality/bulk', + ASSET_CRITICALITY_BULK_URL, { method: 'POST', body: JSON.stringify({ records: assetCriticalityRecords }), }, - version, + { apiVersion: API_VERSIONS.public.v1, space }, ); }; @@ -136,9 +162,8 @@ export const createRule = ({ from?: string; interval?: string; } = {}): Promise<{ id: string; name: string }> => { - const url = DETECTION_ENGINE_RULES_URL(space); return kibanaFetch<{ id: string; name: string }>( - url, + DETECTION_ENGINE_RULES_URL, { method: 'POST', body: JSON.stringify({ @@ -155,19 +180,19 @@ export const createRule = ({ interval: interval || '1m', }), }, - '2023-10-31', + { apiVersion: API_VERSIONS.public.v1, space }, ); }; export const getRule = async (ruleId: string, space?: string) => { - const url = DETECTION_ENGINE_RULES_URL(space); + const url = DETECTION_ENGINE_RULES_URL + '?rule_id=' + ruleId; try { return await kibanaFetch( - url + '?rule_id=' + ruleId, + url, { method: 'GET', }, - '2023-10-31', + { apiVersion: API_VERSIONS.public.v1, space }, ); // eslint-disable-next-line @typescript-eslint/no-unused-vars } catch (e) { @@ -176,16 +201,16 @@ export const getRule = async (ruleId: string, space?: string) => { }; export const deleteRule = async (ruleId: string, space?: string) => { - const url = DETECTION_ENGINE_RULES_URL(space); + const url = DETECTION_ENGINE_RULES_URL + '?rule_id=' + ruleId; return kibanaFetch( - url + '?rule_id=' + ruleId, + url, { method: 'DELETE', body: JSON.stringify({ rule_id: ruleId, }), }, - '2023-10-31', + { apiVersion: API_VERSIONS.public.v1, space }, ); }; @@ -198,10 +223,8 @@ export const createComponentTemplate = async ({ mappings: object; space?: string; }) => { - const url = COMPONENT_TEMPLATES_URL(space); - const ignoreStatus = 409; return kibanaFetch( - url, + COMPONENT_TEMPLATES_URL, { method: 'POST', body: JSON.stringify({ @@ -215,8 +238,7 @@ export const createComponentTemplate = async ({ }, }), }, - '2023-10-31', - ignoreStatus, + { apiVersion: API_VERSIONS.public.v1, ignoreStatuses: [409], space }, ); }; export const installPackage = async ({ @@ -228,14 +250,14 @@ export const installPackage = async ({ version?: string; space?: string; }) => { - const url = FLEET_EPM_PACKAGES_URL(packageName, version, space); + const url = FLEET_EPM_PACKAGES_URL(packageName, version); return kibanaFetch( url, { method: 'POST', }, - '2023-10-31', + { apiVersion: API_VERSIONS.public.v1, space }, ); }; @@ -275,22 +297,32 @@ export const installLegacyRiskScore = async () => { }; export const createSpace = async (space: string) => { - return kibanaFetch(SPACES_URL, { - method: 'POST', - body: JSON.stringify({ - id: space, - name: space, - description: 'Created by security-documents-generator for testing', - disabledFeatures: [], - }), - }); + return kibanaFetch( + SPACES_URL, + { + method: 'POST', + body: JSON.stringify({ + id: space, + name: space, + description: 'Created by security-documents-generator for testing', + disabledFeatures: [], + }), + }, + { + apiVersion: API_VERSIONS.public.v1, + }, + ); }; -export const getSpace = async (space: string): Promise => { +export const doesSpaceExist = async (space: string): Promise => { try { - await kibanaFetch(SPACE_URL(space), { - method: 'GET', - }); + await kibanaFetch( + SPACE_URL(space), + { + method: 'GET', + }, + { apiVersion: API_VERSIONS.public.v1 }, + ); // eslint-disable-next-line @typescript-eslint/no-unused-vars } catch (e) { return false; @@ -299,31 +331,23 @@ export const getSpace = async (space: string): Promise => { }; const _initEngine = (engineType: string, space?: string) => { - const url = space - ? `/s/${space}/api/entity_store/engines/${engineType}/init` - : `/api/entity_store/engines/${engineType}/init`; - return kibanaFetch( - url, + INIT_ENTITY_ENGINE_URL(engineType), { method: 'POST', body: JSON.stringify({}), }, - '2023-10-31', + { apiVersion: API_VERSIONS.public.v1, space }, ); }; const _deleteEngine = (engineType: string, space?: string) => { - const url = space - ? `/s/${space}/api/entity_store/engines/${engineType}` - : `/api/entity_store/engines/${engineType}`; - return kibanaFetch( - url, + ENTITY_ENGINE_URL(engineType), { method: 'DELETE', }, - '2023-10-31', + { apiVersion: API_VERSIONS.public.v1, space }, ); }; @@ -338,16 +362,12 @@ export const deleteEngines = async ( }; const _listEngines = (space?: string) => { - const url = space - ? `/s/${space}/api/entity_store/engines` - : '/api/entity_store/engines'; - const res = kibanaFetch( - url, + ENTITY_ENGINES_URL, { method: 'GET', }, - '2023-10-31', + { apiVersion: API_VERSIONS.public.v1, space }, ); return res as Promise<{ engines: Array<{ status: string }> }>; @@ -388,22 +408,23 @@ export const initEntityEngineForEntityTypes = async ( }; export const getAllRules = async (space?: string) => { - const url = DETECTION_ENGINE_RULES_URL(space); const perPage = 100; // Maximum items per page let page = 1; let allRules: Array<{ rule_id: string; name: string; id: string }> = []; try { while (true) { + const url = + DETECTION_ENGINE_RULES_URL + `/_find?page=${page}&per_page=${perPage}`; const response = await kibanaFetch<{ data: Array<{ rule_id: string; name: string; id: string }>; total: number; }>( - url + `/_find?page=${page}&per_page=${perPage}`, + url, { method: 'GET', }, - '2023-10-31', + { apiVersion: API_VERSIONS.public.v1, space }, ); if (!response.data || response.data.length === 0) { @@ -428,9 +449,8 @@ export const getAllRules = async (space?: string) => { }; export const bulkDeleteRules = async (ruleIds: string[], space?: string) => { - const url = DETECTION_ENGINE_RULES_URL(space) + '/_bulk_action'; return kibanaFetch( - url, + DETECTION_ENGINE_RULES_BULK_ACTION_URL, { method: 'POST', body: JSON.stringify({ @@ -438,6 +458,6 @@ export const bulkDeleteRules = async (ruleIds: string[], space?: string) => { ids: ruleIds, }), }, - '2023-10-31', + { apiVersion: API_VERSIONS.public.v1, space }, ); }; From 9046eaa98919c97d87d01fd9c0e50b0f6f632fab Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 3 Mar 2025 11:18:35 +0000 Subject: [PATCH 2/3] Update src/commands/asset_criticality.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/commands/asset_criticality.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/asset_criticality.ts b/src/commands/asset_criticality.ts index ef07a00..060d151 100644 --- a/src/commands/asset_criticality.ts +++ b/src/commands/asset_criticality.ts @@ -13,7 +13,7 @@ export const generateAssetCriticality = async ({ users, hosts, seed = generateNewSeed(), - space, + space = 'default', }: { users: number; hosts: number; From 1ea00c7a7561c5f690bf409d078ce969f165f43a Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 3 Mar 2025 12:06:45 +0000 Subject: [PATCH 3/3] remove incrorrect body --- src/utils/kibana_api.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/utils/kibana_api.ts b/src/utils/kibana_api.ts index ae1bf4d..f312c0e 100644 --- a/src/utils/kibana_api.ts +++ b/src/utils/kibana_api.ts @@ -206,9 +206,6 @@ export const deleteRule = async (ruleId: string, space?: string) => { url, { method: 'DELETE', - body: JSON.stringify({ - rule_id: ruleId, - }), }, { apiVersion: API_VERSIONS.public.v1, space }, );