From 7a3e4b973382bc1cf16711f6702f25636dda141a Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 21 Jan 2025 01:29:27 +1100 Subject: [PATCH] [8.x] [Discover] Add selector syntax support to log source profile (#206937) (#207190) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Backport This will backport the following commits from `main` to `8.x`: - [[Discover] Add selector syntax support to log source profile (#206937)](/~https://github.com/elastic/kibana/pull/206937) ### Questions ? Please refer to the [Backport tool documentation](/~https://github.com/sqren/backport) Co-authored-by: Felix Stürmer --- .../utils/create_regexp_pattern_from.test.ts | 48 ++++++- .../src/utils/create_regexp_pattern_from.ts | 35 +++-- .../data_types/logs/logs_context_service.ts | 3 +- .../logs_data_source_profile/profile.test.ts | 127 ++++++++++-------- .../sub_profiles/create_resolve.ts | 2 +- .../data_views/models/data_view_descriptor.ts | 4 +- 6 files changed, 147 insertions(+), 72 deletions(-) diff --git a/src/platform/packages/shared/kbn-data-view-utils/src/utils/create_regexp_pattern_from.test.ts b/src/platform/packages/shared/kbn-data-view-utils/src/utils/create_regexp_pattern_from.test.ts index ea4b5a0c77a75..821e8b5a3d79a 100644 --- a/src/platform/packages/shared/kbn-data-view-utils/src/utils/create_regexp_pattern_from.test.ts +++ b/src/platform/packages/shared/kbn-data-view-utils/src/utils/create_regexp_pattern_from.test.ts @@ -10,7 +10,7 @@ import { createRegExpPatternFrom } from './create_regexp_pattern_from'; describe('createRegExpPatternFrom should create a regular expression starting from a string that', () => { - const regExpPattern = createRegExpPatternFrom('logs'); + const regExpPattern = createRegExpPatternFrom('logs', 'data'); it('tests positive for single index patterns starting with the passed base pattern', () => { expect('logs*').toMatch(regExpPattern); @@ -62,6 +62,52 @@ describe('createRegExpPatternFrom should create a regular expression starting fr expect('cluster1:logs-*,logs-*,').toMatch(regExpPattern); }); + it('tests correctly for patterns with the data selector suffix', () => { + expect('logs-*::data').toMatch(createRegExpPatternFrom('logs', 'data')); + expect('logs-*::data,').toMatch(createRegExpPatternFrom('logs', 'data')); + expect('cluster1:logs-*::data,logs-*::data').toMatch(createRegExpPatternFrom('logs', 'data')); + + expect('logs-*').not.toMatch(createRegExpPatternFrom('logs', 'failures')); + expect('logs-*::data').not.toMatch(createRegExpPatternFrom('logs', 'failures')); + expect('cluster1:logs-*::data,logs-*::data').not.toMatch( + createRegExpPatternFrom('logs', 'failures') + ); + + expect('logs-*').not.toMatch(createRegExpPatternFrom('logs', '*')); + expect('logs-*::data').not.toMatch(createRegExpPatternFrom('logs', '*')); + expect('cluster1:logs-*::data,logs-*::data').not.toMatch(createRegExpPatternFrom('logs', '*')); + }); + + it('tests correctly for patterns with the failures selector suffix', () => { + expect('logs-*::failures').toMatch(createRegExpPatternFrom('logs', 'failures')); + expect('logs-*::failures,').toMatch(createRegExpPatternFrom('logs', 'failures')); + expect('cluster1:logs-*::failures,logs-*::failures').toMatch( + createRegExpPatternFrom('logs', 'failures') + ); + + expect('logs-*::failures').not.toMatch(createRegExpPatternFrom('logs', 'data')); + expect('cluster1:logs-*::failures,logs-*::failures').not.toMatch( + createRegExpPatternFrom('logs', 'data') + ); + + expect('logs-*::failures').not.toMatch(createRegExpPatternFrom('logs', '*')); + expect('cluster1:logs-*::failures,logs-*::failures').not.toMatch( + createRegExpPatternFrom('logs', '*') + ); + }); + + it('tests correctly for patterns with the wildcard selector suffix', () => { + expect('logs-*::*').toMatch(createRegExpPatternFrom('logs', '*')); + expect('logs-*::*,').toMatch(createRegExpPatternFrom('logs', '*')); + expect('cluster1:logs-*::*,logs-*::*').toMatch(createRegExpPatternFrom('logs', '*')); + + expect('logs-*::*').not.toMatch(createRegExpPatternFrom('logs', 'data')); + expect('cluster1:logs-*::*,logs-*::*').not.toMatch(createRegExpPatternFrom('logs', 'data')); + + expect('logs-*::*').not.toMatch(createRegExpPatternFrom('logs', 'failures')); + expect('cluster1:logs-*::*,logs-*::*').not.toMatch(createRegExpPatternFrom('logs', 'failures')); + }); + it('tests negative for patterns with spaces and unexpected commas', () => { expect('cluster1:logs-*,clust,er2:logs-*').not.toMatch(regExpPattern); expect('cluster1:logs-*, cluster2:logs-*').not.toMatch(regExpPattern); diff --git a/src/platform/packages/shared/kbn-data-view-utils/src/utils/create_regexp_pattern_from.ts b/src/platform/packages/shared/kbn-data-view-utils/src/utils/create_regexp_pattern_from.ts index 35bb80612458c..b3d05b34737de 100644 --- a/src/platform/packages/shared/kbn-data-view-utils/src/utils/create_regexp_pattern_from.ts +++ b/src/platform/packages/shared/kbn-data-view-utils/src/utils/create_regexp_pattern_from.ts @@ -7,12 +7,31 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export const createRegExpPatternFrom = (basePatterns: string | string[]) => { - const patterns = Array.isArray(basePatterns) ? basePatterns : [basePatterns]; - // Create the base patterns union with strict boundaries - const basePatternGroup = `[^,\\s]*(\\b|_)(${patterns.join('|')})(\\b|_)([^,\\s]*)?`; - // Apply base patterns union for local and remote clusters - const localAndRemotePatternGroup = `((${basePatternGroup})|([^:,\\s]*:${basePatternGroup}))`; - // Handle trailing comma and multiple pattern concatenation - return new RegExp(`^${localAndRemotePatternGroup}(,${localAndRemotePatternGroup})*(,$|$)`, 'i'); +import { escapeRegExp } from 'lodash'; + +type Selector = 'data' | 'failures' | '*'; + +export const createRegExpPatternFrom = (basePatterns: string | string[], selector: Selector) => { + const normalizedBasePatterns = normalizeBasePatterns(basePatterns); + + const indexNames = `(?:${normalizedBasePatterns.join('|')})`; + const selectorsSuffix = `(?:::(?:${escapeRegExp(selector)}))${ + isDefaultSelector(selector) ? '?' : '' + }`; + + return new RegExp( + `^(?:${optionalRemoteCluster}${optionalIndexNamePrefix}${indexNames}${optionalIndexNameSuffix}${selectorsSuffix},?)+$`, + 'i' + ); }; + +const normalizeBasePatterns = (basePatterns: string | string[]): string[] => + (Array.isArray(basePatterns) ? basePatterns : [basePatterns]).map(escapeRegExp); + +const isDefaultSelector = (selector: Selector): boolean => selector === 'data'; + +const nameCharacters = '[^:,\\s]+'; +const segmentBoundary = '(?:\\b|_)'; +const optionalRemoteCluster = `(?:${nameCharacters}:)?`; +const optionalIndexNamePrefix = `(?:${nameCharacters}${segmentBoundary})?`; +const optionalIndexNameSuffix = `(?:${segmentBoundary}${nameCharacters})?`; diff --git a/src/platform/packages/shared/kbn-discover-utils/src/data_types/logs/logs_context_service.ts b/src/platform/packages/shared/kbn-discover-utils/src/data_types/logs/logs_context_service.ts index 5f05def6d9c94..618ac31c6f1e5 100644 --- a/src/platform/packages/shared/kbn-discover-utils/src/data_types/logs/logs_context_service.ts +++ b/src/platform/packages/shared/kbn-discover-utils/src/data_types/logs/logs_context_service.ts @@ -28,7 +28,8 @@ export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS = [ ]; export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP = createRegExpPatternFrom( - DEFAULT_ALLOWED_LOGS_BASE_PATTERNS + DEFAULT_ALLOWED_LOGS_BASE_PATTERNS, + 'data' ); export const createLogsContextService = async ({ logsDataAccess }: LogsContextServiceDeps) => { diff --git a/src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/profile.test.ts b/src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/profile.test.ts index 7740602203e5a..1e84012942c16 100644 --- a/src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/profile.test.ts +++ b/src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/profile.test.ts @@ -28,9 +28,20 @@ const mockServices = createContextAwarenessMocks().profileProviderServices; describe('logsDataSourceProfileProvider', () => { const logsDataSourceProfileProvider = createLogsDataSourceProfileProvider(mockServices); - const VALID_INDEX_PATTERN = 'logs-nginx.access-*'; - const MIXED_INDEX_PATTERN = 'logs-nginx.access-*,metrics-*'; - const INVALID_INDEX_PATTERN = 'my_source-access-*'; + + const VALID_IMPLICIT_DATA_INDEX_PATTERN = 'logs-nginx.access-*'; + const VALID_INDEX_PATTERNS: Array<[string, string]> = [ + ['explicit data', 'logs-nginx.access-*::data'], + ['implicit data', VALID_IMPLICIT_DATA_INDEX_PATTERN], + ['mixed data selector qualification', 'logs-nginx.access-*::data,logs-nginx.error-*'], + ]; + const INVALID_INDEX_PATTERNS: Array<[string, string]> = [ + ['forbidden implicit data', 'my_source-access-*'], + ['mixed implicit data', 'logs-nginx.access-*,metrics-*'], + ['mixed explicit data', 'logs-nginx.access-*::data,metrics-*::data'], + ['mixed selector', 'logs-nginx.access-*,logs-nginx.access-*::failures'], + ]; + const ROOT_CONTEXT: ContextWithProfileId = { profileId: OBSERVABILITY_ROOT_PROFILE_ID, solutionType: SolutionType.Observability, @@ -43,64 +54,62 @@ describe('logsDataSourceProfileProvider', () => { isMatch: false, }; - it('should match ES|QL sources with an allowed index pattern in its query', () => { - expect( - logsDataSourceProfileProvider.resolve({ - rootContext: ROOT_CONTEXT, - dataSource: createEsqlDataSource(), - query: { esql: `from ${VALID_INDEX_PATTERN}` }, - }) - ).toEqual(RESOLUTION_MATCH); - }); - - it('should NOT match ES|QL sources with a mixed or not allowed index pattern in its query', () => { - expect( - logsDataSourceProfileProvider.resolve({ - rootContext: ROOT_CONTEXT, - dataSource: createEsqlDataSource(), - query: { esql: `from ${INVALID_INDEX_PATTERN}` }, - }) - ).toEqual(RESOLUTION_MISMATCH); - expect( - logsDataSourceProfileProvider.resolve({ - rootContext: ROOT_CONTEXT, - dataSource: createEsqlDataSource(), - query: { esql: `from ${MIXED_INDEX_PATTERN}` }, - }) - ).toEqual(RESOLUTION_MISMATCH); - }); - - it('should match data view sources with an allowed index pattern', () => { - expect( - logsDataSourceProfileProvider.resolve({ - rootContext: ROOT_CONTEXT, - dataSource: createDataViewDataSource({ dataViewId: VALID_INDEX_PATTERN }), - dataView: createStubIndexPattern({ spec: { title: VALID_INDEX_PATTERN } }), - }) - ).toEqual(RESOLUTION_MATCH); - }); - - it('should NOT match data view sources with a mixed or not allowed index pattern', () => { - expect( - logsDataSourceProfileProvider.resolve({ - rootContext: ROOT_CONTEXT, - dataSource: createDataViewDataSource({ dataViewId: INVALID_INDEX_PATTERN }), - dataView: createStubIndexPattern({ spec: { title: INVALID_INDEX_PATTERN } }), - }) - ).toEqual(RESOLUTION_MISMATCH); - expect( - logsDataSourceProfileProvider.resolve({ - rootContext: ROOT_CONTEXT, - dataSource: createDataViewDataSource({ dataViewId: MIXED_INDEX_PATTERN }), - dataView: createStubIndexPattern({ spec: { title: MIXED_INDEX_PATTERN } }), - }) - ).toEqual(RESOLUTION_MISMATCH); - }); + it.each(VALID_INDEX_PATTERNS)( + 'should match ES|QL sources with an allowed %s index pattern in its query', + (_, validIndexPattern) => { + expect( + logsDataSourceProfileProvider.resolve({ + rootContext: ROOT_CONTEXT, + dataSource: createEsqlDataSource(), + query: { esql: `from ${validIndexPattern}` }, + }) + ).toEqual(RESOLUTION_MATCH); + } + ); + + it.each(INVALID_INDEX_PATTERNS)( + 'should NOT match ES|QL sources with a %s index pattern in its query', + (_, invalidIndexPattern) => { + expect( + logsDataSourceProfileProvider.resolve({ + rootContext: ROOT_CONTEXT, + dataSource: createEsqlDataSource(), + query: { esql: `from ${invalidIndexPattern}` }, + }) + ).toEqual(RESOLUTION_MISMATCH); + } + ); + + it.each(VALID_INDEX_PATTERNS)( + 'should match data view sources with an allowed %s index pattern', + (_, validIndexPattern) => { + expect( + logsDataSourceProfileProvider.resolve({ + rootContext: ROOT_CONTEXT, + dataSource: createDataViewDataSource({ dataViewId: validIndexPattern }), + dataView: createStubIndexPattern({ spec: { title: validIndexPattern } }), + }) + ).toEqual(RESOLUTION_MATCH); + } + ); + + it.each(INVALID_INDEX_PATTERNS)( + 'should NOT match data view sources with a %s index pattern', + (_, invalidIndexPattern) => { + expect( + logsDataSourceProfileProvider.resolve({ + rootContext: ROOT_CONTEXT, + dataSource: createDataViewDataSource({ dataViewId: invalidIndexPattern }), + dataView: createStubIndexPattern({ spec: { title: invalidIndexPattern } }), + }) + ).toEqual(RESOLUTION_MISMATCH); + } + ); it('does NOT match data view sources when solution type is not Observability', () => { const params: Omit = { dataSource: createEsqlDataSource(), - query: { esql: `from ${VALID_INDEX_PATTERN}` }, + query: { esql: `from ${VALID_IMPLICIT_DATA_INDEX_PATTERN}` }, }; expect(logsDataSourceProfileProvider.resolve({ ...params, rootContext: ROOT_CONTEXT })).toEqual( RESOLUTION_MATCH @@ -127,7 +136,7 @@ describe('logsDataSourceProfileProvider', () => { const dataViewWithLogLevel = createStubIndexPattern({ spec: { - title: VALID_INDEX_PATTERN, + title: VALID_IMPLICIT_DATA_INDEX_PATTERN, fields: { 'log.level': { name: 'log.level', @@ -146,7 +155,7 @@ describe('logsDataSourceProfileProvider', () => { const dataViewWithoutLogLevel = createStubIndexPattern({ spec: { - title: VALID_INDEX_PATTERN, + title: VALID_IMPLICIT_DATA_INDEX_PATTERN, }, }); diff --git a/src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/sub_profiles/create_resolve.ts b/src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/sub_profiles/create_resolve.ts index fb2afd46066ca..822d8c5e065eb 100644 --- a/src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/sub_profiles/create_resolve.ts +++ b/src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/sub_profiles/create_resolve.ts @@ -14,7 +14,7 @@ import { OBSERVABILITY_ROOT_PROFILE_ID } from '../../consts'; export const createResolve = (baseIndexPattern: string): DataSourceProfileProvider['resolve'] => { const testIndexPattern = testPatternAgainstAllowedList([ - createRegExpPatternFrom(baseIndexPattern), + createRegExpPatternFrom(baseIndexPattern, 'data'), ]); return (params) => { diff --git a/x-pack/solutions/observability/plugins/logs_explorer/common/data_views/models/data_view_descriptor.ts b/x-pack/solutions/observability/plugins/logs_explorer/common/data_views/models/data_view_descriptor.ts index b5fe3d1f58c0f..c21e97212835c 100644 --- a/x-pack/solutions/observability/plugins/logs_explorer/common/data_views/models/data_view_descriptor.ts +++ b/x-pack/solutions/observability/plugins/logs_explorer/common/data_views/models/data_view_descriptor.ts @@ -11,7 +11,7 @@ import { DataViewSpecWithId } from '../../data_source_selection'; import { DataViewDescriptorType } from '../types'; const LOGS_ALLOWED_LIST = [ - createRegExpPatternFrom(DEFAULT_ALLOWED_LOGS_BASE_PATTERNS), + createRegExpPatternFrom(DEFAULT_ALLOWED_LOGS_BASE_PATTERNS, 'data'), // Add more strings or regex patterns as needed ]; @@ -59,7 +59,7 @@ export class DataViewDescriptor { testAgainstAllowedList(allowedList: string[]) { return this.title - ? testPatternAgainstAllowedList([createRegExpPatternFrom(allowedList)])(this.title) + ? testPatternAgainstAllowedList([createRegExpPatternFrom(allowedList, 'data')])(this.title) : false; }