Skip to content

Commit

Permalink
[8.x] [Discover] Add selector syntax support to log source profile (#…
Browse files Browse the repository at this point in the history 
…206937) (#207190)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Discover] Add selector syntax support to log source profile
(#206937)](#206937)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](/~https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Felix
Stürmer","email":"weltenwort@users.noreply.github.com"},"sourceCommit":{"committedDate":"2025-01-20T12:50:32Z","message":"[Discover]
Add selector syntax support to log source profile (#206937)\n\nThis adds
support for the new selector syntax to the log source
profile\r\nheuristics. It will only match when index name expression
exclusively\r\ncontains implicit or explicit `data`
selectors.\r\n\r\nCo-authored-by: Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"032c481ec17e5b4bed83830a34a5c00834ec7116","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:enhancement","v9.0.0","backport:prev-minor","Team:obs-ux-logs"],"title":"[Discover]
Add selector syntax support to log source
profile","number":206937,"url":"/~https://github.com/elastic/kibana/pull/206937","mergeCommit":{"message":"[Discover]
Add selector syntax support to log source profile (#206937)\n\nThis adds
support for the new selector syntax to the log source
profile\r\nheuristics. It will only match when index name expression
exclusively\r\ncontains implicit or explicit `data`
selectors.\r\n\r\nCo-authored-by: Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"032c481ec17e5b4bed83830a34a5c00834ec7116"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"/~https://github.com/elastic/kibana/pull/206937","number":206937,"mergeCommit":{"message":"[Discover]
Add selector syntax support to log source profile (#206937)\n\nThis adds
support for the new selector syntax to the log source
profile\r\nheuristics. It will only match when index name expression
exclusively\r\ncontains implicit or explicit `data`
selectors.\r\n\r\nCo-authored-by: Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"032c481ec17e5b4bed83830a34a5c00834ec7116"}}]}]
BACKPORT-->

Co-authored-by: Felix Stürmer <weltenwort@users.noreply.github.com>
  • Loading branch information
@kibanamachine @weltenwort
kibanamachine and weltenwort authored Jan 20, 2025
1 parent 491c19e commit 7a3e4b9
Show file tree
Hide file tree
Showing 6 changed files with 147 additions and 72 deletions.
48 changes: 47 additions & 1 deletion ...platform/packages/shared/kbn-data-view-utils/src/utils/create_regexp_pattern_from.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import { createRegExpPatternFrom } from './create_regexp_pattern_from';

describe('createRegExpPatternFrom should create a regular expression starting from a string that', () => {
const regExpPattern = createRegExpPatternFrom('logs');
const regExpPattern = createRegExpPatternFrom('logs', 'data');

it('tests positive for single index patterns starting with the passed base pattern', () => {
expect('logs*').toMatch(regExpPattern);
Expand Down Expand Up @@ -62,6 +62,52 @@ describe('createRegExpPatternFrom should create a regular expression starting fr
expect('cluster1:logs-*,logs-*,').toMatch(regExpPattern);
});

it('tests correctly for patterns with the data selector suffix', () => {
expect('logs-*::data').toMatch(createRegExpPatternFrom('logs', 'data'));
expect('logs-*::data,').toMatch(createRegExpPatternFrom('logs', 'data'));
expect('cluster1:logs-*::data,logs-*::data').toMatch(createRegExpPatternFrom('logs', 'data'));

expect('logs-*').not.toMatch(createRegExpPatternFrom('logs', 'failures'));
expect('logs-*::data').not.toMatch(createRegExpPatternFrom('logs', 'failures'));
expect('cluster1:logs-*::data,logs-*::data').not.toMatch(
createRegExpPatternFrom('logs', 'failures')
);

expect('logs-*').not.toMatch(createRegExpPatternFrom('logs', '*'));
expect('logs-*::data').not.toMatch(createRegExpPatternFrom('logs', '*'));
expect('cluster1:logs-*::data,logs-*::data').not.toMatch(createRegExpPatternFrom('logs', '*'));
});

it('tests correctly for patterns with the failures selector suffix', () => {
expect('logs-*::failures').toMatch(createRegExpPatternFrom('logs', 'failures'));
expect('logs-*::failures,').toMatch(createRegExpPatternFrom('logs', 'failures'));
expect('cluster1:logs-*::failures,logs-*::failures').toMatch(
createRegExpPatternFrom('logs', 'failures')
);

expect('logs-*::failures').not.toMatch(createRegExpPatternFrom('logs', 'data'));
expect('cluster1:logs-*::failures,logs-*::failures').not.toMatch(
createRegExpPatternFrom('logs', 'data')
);

expect('logs-*::failures').not.toMatch(createRegExpPatternFrom('logs', '*'));
expect('cluster1:logs-*::failures,logs-*::failures').not.toMatch(
createRegExpPatternFrom('logs', '*')
);
});

it('tests correctly for patterns with the wildcard selector suffix', () => {
expect('logs-*::*').toMatch(createRegExpPatternFrom('logs', '*'));
expect('logs-*::*,').toMatch(createRegExpPatternFrom('logs', '*'));
expect('cluster1:logs-*::*,logs-*::*').toMatch(createRegExpPatternFrom('logs', '*'));

expect('logs-*::*').not.toMatch(createRegExpPatternFrom('logs', 'data'));
expect('cluster1:logs-*::*,logs-*::*').not.toMatch(createRegExpPatternFrom('logs', 'data'));

expect('logs-*::*').not.toMatch(createRegExpPatternFrom('logs', 'failures'));
expect('cluster1:logs-*::*,logs-*::*').not.toMatch(createRegExpPatternFrom('logs', 'failures'));
});

it('tests negative for patterns with spaces and unexpected commas', () => {
expect('cluster1:logs-*,clust,er2:logs-*').not.toMatch(regExpPattern);
expect('cluster1:logs-*, cluster2:logs-*').not.toMatch(regExpPattern);
Expand Down
35 changes: 27 additions & 8 deletions src/platform/packages/shared/kbn-data-view-utils/src/utils/create_regexp_pattern_from.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,31 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/

export const createRegExpPatternFrom = (basePatterns: string | string[]) => {
const patterns = Array.isArray(basePatterns) ? basePatterns : [basePatterns];
// Create the base patterns union with strict boundaries
const basePatternGroup = `[^,\\s]*(\\b|_)(${patterns.join('|')})(\\b|_)([^,\\s]*)?`;
// Apply base patterns union for local and remote clusters
const localAndRemotePatternGroup = `((${basePatternGroup})|([^:,\\s]*:${basePatternGroup}))`;
// Handle trailing comma and multiple pattern concatenation
return new RegExp(`^${localAndRemotePatternGroup}(,${localAndRemotePatternGroup})*(,$|$)`, 'i');
import { escapeRegExp } from 'lodash';

type Selector = 'data' | 'failures' | '*';

export const createRegExpPatternFrom = (basePatterns: string | string[], selector: Selector) => {
const normalizedBasePatterns = normalizeBasePatterns(basePatterns);

const indexNames = `(?:${normalizedBasePatterns.join('|')})`;
const selectorsSuffix = `(?:::(?:${escapeRegExp(selector)}))${
isDefaultSelector(selector) ? '?' : ''
}`;

return new RegExp(
`^(?:${optionalRemoteCluster}${optionalIndexNamePrefix}${indexNames}${optionalIndexNameSuffix}${selectorsSuffix},?)+$`,
'i'
);
};

const normalizeBasePatterns = (basePatterns: string | string[]): string[] =>
(Array.isArray(basePatterns) ? basePatterns : [basePatterns]).map(escapeRegExp);

const isDefaultSelector = (selector: Selector): boolean => selector === 'data';

const nameCharacters = '[^:,\\s]+';
const segmentBoundary = '(?:\\b|_)';
const optionalRemoteCluster = `(?:${nameCharacters}:)?`;
const optionalIndexNamePrefix = `(?:${nameCharacters}${segmentBoundary})?`;
const optionalIndexNameSuffix = `(?:${segmentBoundary}${nameCharacters})?`;
3 changes: 2 additions & 1 deletion src/platform/packages/shared/kbn-discover-utils/src/data_types/logs/logs_context_service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS = [
];

export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP = createRegExpPatternFrom(
DEFAULT_ALLOWED_LOGS_BASE_PATTERNS
DEFAULT_ALLOWED_LOGS_BASE_PATTERNS,
'data'
);

export const createLogsContextService = async ({ logsDataAccess }: LogsContextServiceDeps) => {
Expand Down
127 changes: 68 additions & 59 deletions ...ontext_awareness/profile_providers/observability/logs_data_source_profile/profile.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,20 @@ const mockServices = createContextAwarenessMocks().profileProviderServices;

describe('logsDataSourceProfileProvider', () => {
const logsDataSourceProfileProvider = createLogsDataSourceProfileProvider(mockServices);
const VALID_INDEX_PATTERN = 'logs-nginx.access-*';
const MIXED_INDEX_PATTERN = 'logs-nginx.access-*,metrics-*';
const INVALID_INDEX_PATTERN = 'my_source-access-*';

const VALID_IMPLICIT_DATA_INDEX_PATTERN = 'logs-nginx.access-*';
const VALID_INDEX_PATTERNS: Array<[string, string]> = [
['explicit data', 'logs-nginx.access-*::data'],
['implicit data', VALID_IMPLICIT_DATA_INDEX_PATTERN],
['mixed data selector qualification', 'logs-nginx.access-*::data,logs-nginx.error-*'],
];
const INVALID_INDEX_PATTERNS: Array<[string, string]> = [
['forbidden implicit data', 'my_source-access-*'],
['mixed implicit data', 'logs-nginx.access-*,metrics-*'],
['mixed explicit data', 'logs-nginx.access-*::data,metrics-*::data'],
['mixed selector', 'logs-nginx.access-*,logs-nginx.access-*::failures'],
];

const ROOT_CONTEXT: ContextWithProfileId<RootContext> = {
profileId: OBSERVABILITY_ROOT_PROFILE_ID,
solutionType: SolutionType.Observability,
Expand All @@ -43,64 +54,62 @@ describe('logsDataSourceProfileProvider', () => {
isMatch: false,
};

it('should match ES|QL sources with an allowed index pattern in its query', () => {
expect(
logsDataSourceProfileProvider.resolve({
rootContext: ROOT_CONTEXT,
dataSource: createEsqlDataSource(),
query: { esql: `from ${VALID_INDEX_PATTERN}` },
})
).toEqual(RESOLUTION_MATCH);
});

it('should NOT match ES|QL sources with a mixed or not allowed index pattern in its query', () => {
expect(
logsDataSourceProfileProvider.resolve({
rootContext: ROOT_CONTEXT,
dataSource: createEsqlDataSource(),
query: { esql: `from ${INVALID_INDEX_PATTERN}` },
})
).toEqual(RESOLUTION_MISMATCH);
expect(
logsDataSourceProfileProvider.resolve({
rootContext: ROOT_CONTEXT,
dataSource: createEsqlDataSource(),
query: { esql: `from ${MIXED_INDEX_PATTERN}` },
})
).toEqual(RESOLUTION_MISMATCH);
});

it('should match data view sources with an allowed index pattern', () => {
expect(
logsDataSourceProfileProvider.resolve({
rootContext: ROOT_CONTEXT,
dataSource: createDataViewDataSource({ dataViewId: VALID_INDEX_PATTERN }),
dataView: createStubIndexPattern({ spec: { title: VALID_INDEX_PATTERN } }),
})
).toEqual(RESOLUTION_MATCH);
});

it('should NOT match data view sources with a mixed or not allowed index pattern', () => {
expect(
logsDataSourceProfileProvider.resolve({
rootContext: ROOT_CONTEXT,
dataSource: createDataViewDataSource({ dataViewId: INVALID_INDEX_PATTERN }),
dataView: createStubIndexPattern({ spec: { title: INVALID_INDEX_PATTERN } }),
})
).toEqual(RESOLUTION_MISMATCH);
expect(
logsDataSourceProfileProvider.resolve({
rootContext: ROOT_CONTEXT,
dataSource: createDataViewDataSource({ dataViewId: MIXED_INDEX_PATTERN }),
dataView: createStubIndexPattern({ spec: { title: MIXED_INDEX_PATTERN } }),
})
).toEqual(RESOLUTION_MISMATCH);
});
it.each(VALID_INDEX_PATTERNS)(
'should match ES|QL sources with an allowed %s index pattern in its query',
(_, validIndexPattern) => {
expect(
logsDataSourceProfileProvider.resolve({
rootContext: ROOT_CONTEXT,
dataSource: createEsqlDataSource(),
query: { esql: `from ${validIndexPattern}` },
})
).toEqual(RESOLUTION_MATCH);
}
);

it.each(INVALID_INDEX_PATTERNS)(
'should NOT match ES|QL sources with a %s index pattern in its query',
(_, invalidIndexPattern) => {
expect(
logsDataSourceProfileProvider.resolve({
rootContext: ROOT_CONTEXT,
dataSource: createEsqlDataSource(),
query: { esql: `from ${invalidIndexPattern}` },
})
).toEqual(RESOLUTION_MISMATCH);
}
);

it.each(VALID_INDEX_PATTERNS)(
'should match data view sources with an allowed %s index pattern',
(_, validIndexPattern) => {
expect(
logsDataSourceProfileProvider.resolve({
rootContext: ROOT_CONTEXT,
dataSource: createDataViewDataSource({ dataViewId: validIndexPattern }),
dataView: createStubIndexPattern({ spec: { title: validIndexPattern } }),
})
).toEqual(RESOLUTION_MATCH);
}
);

it.each(INVALID_INDEX_PATTERNS)(
'should NOT match data view sources with a %s index pattern',
(_, invalidIndexPattern) => {
expect(
logsDataSourceProfileProvider.resolve({
rootContext: ROOT_CONTEXT,
dataSource: createDataViewDataSource({ dataViewId: invalidIndexPattern }),
dataView: createStubIndexPattern({ spec: { title: invalidIndexPattern } }),
})
).toEqual(RESOLUTION_MISMATCH);
}
);

it('does NOT match data view sources when solution type is not Observability', () => {
const params: Omit<DataSourceProfileProviderParams, 'rootContext'> = {
dataSource: createEsqlDataSource(),
query: { esql: `from ${VALID_INDEX_PATTERN}` },
query: { esql: `from ${VALID_IMPLICIT_DATA_INDEX_PATTERN}` },
};
expect(logsDataSourceProfileProvider.resolve({ ...params, rootContext: ROOT_CONTEXT })).toEqual(
RESOLUTION_MATCH
Expand All @@ -127,7 +136,7 @@ describe('logsDataSourceProfileProvider', () => {

const dataViewWithLogLevel = createStubIndexPattern({
spec: {
title: VALID_INDEX_PATTERN,
title: VALID_IMPLICIT_DATA_INDEX_PATTERN,
fields: {
'log.level': {
name: 'log.level',
Expand All @@ -146,7 +155,7 @@ describe('logsDataSourceProfileProvider', () => {

const dataViewWithoutLogLevel = createStubIndexPattern({
spec: {
title: VALID_INDEX_PATTERN,
title: VALID_IMPLICIT_DATA_INDEX_PATTERN,
},
});

Expand Down
2 changes: 1 addition & 1 deletion ...s/profile_providers/observability/logs_data_source_profile/sub_profiles/create_resolve.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { OBSERVABILITY_ROOT_PROFILE_ID } from '../../consts';

export const createResolve = (baseIndexPattern: string): DataSourceProfileProvider['resolve'] => {
const testIndexPattern = testPatternAgainstAllowedList([
createRegExpPatternFrom(baseIndexPattern),
createRegExpPatternFrom(baseIndexPattern, 'data'),
]);

return (params) => {
Expand Down
4 changes: 2 additions & 2 deletions ...ions/observability/plugins/logs_explorer/common/data_views/models/data_view_descriptor.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { DataViewSpecWithId } from '../../data_source_selection';
import { DataViewDescriptorType } from '../types';

const LOGS_ALLOWED_LIST = [
createRegExpPatternFrom(DEFAULT_ALLOWED_LOGS_BASE_PATTERNS),
createRegExpPatternFrom(DEFAULT_ALLOWED_LOGS_BASE_PATTERNS, 'data'),
// Add more strings or regex patterns as needed
];

Expand Down Expand Up @@ -59,7 +59,7 @@ export class DataViewDescriptor {

testAgainstAllowedList(allowedList: string[]) {
return this.title
? testPatternAgainstAllowedList([createRegExpPatternFrom(allowedList)])(this.title)
? testPatternAgainstAllowedList([createRegExpPatternFrom(allowedList, 'data')])(this.title)
: false;
}

Expand Down

0 comments on commit 7a3e4b9

Please sign in to comment.