[Security Solution] Implement rule monitoring dashboard (elastic#159875)

**Addresses:** elastic/security-team#6032

## Summary

This PR adds a new `[Elastic Security] Detection rule monitoring` Kibana
dashboard and a new `POST /internal/detection_engine/health/_setup` API
endpoint.

## Dashboard

The dashboard can be helpful for monitoring the health and performance
of Security detection rules. Users of the dashboard must have read
access to the `.kibana-event-log-*` index. The dashboard is
automatically installed into the current Kibana space when a user visits
a page in Security Solution - similar to how we install the Fleet
package with prebuilt detection rules.

<img width="1791" alt="Kibana dashboards page"
src="/~https://github.com/elastic/kibana/assets/7359339/92cb3c75-39ea-4069-b70f-8f531869edf7">

<img width="1775" alt="Security dashboards page"
src="/~https://github.com/elastic/kibana/assets/7359339/3b27aeb6-2222-40fd-a453-c204fcee4f31">

![Rule monitoring dashboard
itself](/~https://github.com/elastic/kibana/assets/7359339/755cc044-5613-4c78-b89f-2a9734ded76d)


## API endpoint

The PR also adds a new endpoint for setting up anything related to
monitoring rules and the health of the Detection Engine. If you call the
endpoint, it will install the new dashboard to the Default Kibana space:

```
POST /internal/detection_engine/health/_setup
```

In order to install the dashboard to a different Kibana space, you will
need to call it like that:

```
POST /s/<space-id>/internal/detection_engine/health/_setup
```

The user calling the endpoint must have access to Security Solution. No
additional privileges are required, because the endpoint installs the
dashboard on behalf of the internal user (`kibana_system`).

### Checklist

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](/~https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
  - [ ] elastic/security-docs#3478
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/urls.ts

@@ -28,6 +28,13 @@ export const GET_SPACE_HEALTH_URL = `${INTERNAL_URL}/health/_space` as const;
  */
 export const GET_RULE_HEALTH_URL = `${INTERNAL_URL}/health/_rule` as const;
 
+/**
+ * Similar to the "setup" command of beats, this endpoint installs resources
+ * (dashboards, data views, etc) related to rule monitoring and Detection Engine health,
+ * and can do any other setup work.
+ */
+export const SETUP_HEALTH_URL = `${INTERNAL_URL}/health/_setup` as const;
+
 // -------------------------------------------------------------------------------------------------
 // Rule execution logs API
 

x-pack/plugins/security_solution/public/app/home/index.tsx

@@ -25,8 +25,9 @@ import { TourContextProvider } from '../../common/components/guided_onboarding_t
 
 import { useUrlState } from '../../common/hooks/use_url_state';
 import { useUpdateBrowserTitle } from '../../common/hooks/use_update_browser_title';
-import { useUpgradeSecurityPackages } from '../../detection_engine/rule_management/logic/use_upgrade_security_packages';
 import { useUpdateExecutionContext } from '../../common/hooks/use_update_execution_context';
+import { useUpgradeSecurityPackages } from '../../detection_engine/rule_management/logic/use_upgrade_security_packages';
+import { useSetupDetectionEngineHealthApi } from '../../detection_engine/rule_monitoring';
 
 interface HomePageProps {
   children: React.ReactNode;
@@ -41,12 +42,14 @@ const HomePageComponent: React.FC<HomePageProps> = ({ children, setHeaderActionM
   useUpdateExecutionContext();
 
   const { browserFields } = useSourcererDataView(getScopeFromPath(pathname));
+
   // side effect: this will attempt to upgrade the endpoint package if it is not up to date
   // this will run when a user navigates to the Security Solution app and when they navigate between
   // tabs in the app. This is useful for keeping the endpoint package as up to date as possible until
   // a background task solution can be built on the server side. Once a background task solution is available we
   // can remove this.
   useUpgradeSecurityPackages();
+  useSetupDetectionEngineHealthApi();
 
   return (
     <SecuritySolutionAppWrapper id="security-solution-app" className="kbnAppWrapper">

x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/api/__mocks__/api_client.ts

@@ -21,6 +21,8 @@ import type {
 } from '../api_client_interface';
 
 export const api: jest.Mocked<IRuleMonitoringApiClient> = {
+  setupDetectionEngineHealthApi: jest.fn<Promise<void>, []>().mockResolvedValue(),
+
   fetchRuleExecutionEvents: jest
     .fn<Promise<GetRuleExecutionEventsResponse>, [FetchRuleExecutionEventsArgs]>()
     .mockResolvedValue({

x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/api/api_client.test.ts

@@ -27,6 +27,23 @@ describe('Rule Monitoring API Client', () => {
 
   const signal = new AbortController().signal;
 
+  describe('setupDetectionEngineHealthApi', () => {
+    const responseMock = {};
+
+    beforeEach(() => {
+      fetchMock.mockClear();
+      fetchMock.mockResolvedValue(responseMock);
+    });
+
+    it('calls API with correct parameters', async () => {
+      await api.setupDetectionEngineHealthApi();
+
+      expect(fetchMock).toHaveBeenCalledWith('/internal/detection_engine/health/_setup', {
+        method: 'POST',
+      });
+    });
+  });
+
   describe('fetchRuleExecutionEvents', () => {
     const responseMock: GetRuleExecutionEventsResponse = {
       events: [],

x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/api/api_client.ts

@@ -16,6 +16,7 @@ import type {
 import {
   getRuleExecutionEventsUrl,
   getRuleExecutionResultsUrl,
+  SETUP_HEALTH_URL,
 } from '../../../../common/detection_engine/rule_monitoring';
 
 import type {
@@ -25,6 +26,12 @@ import type {
 } from './api_client_interface';
 
 export const api: IRuleMonitoringApiClient = {
+  setupDetectionEngineHealthApi: async (): Promise<void> => {
+    await http().fetch(SETUP_HEALTH_URL, {
+      method: 'POST',
+    });
+  },
+
   fetchRuleExecutionEvents: (
     args: FetchRuleExecutionEventsArgs
   ): Promise<GetRuleExecutionEventsResponse> => {

x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/api/api_client_interface.ts

@@ -16,6 +16,12 @@ import type {
 } from '../../../../common/detection_engine/rule_monitoring';
 
 export interface IRuleMonitoringApiClient {
+  /**
+   * Installs resources (dashboards, data views, etc) related to rule monitoring
+   * and Detection Engine health, and can do any other setup work.
+   */
+  setupDetectionEngineHealthApi(): Promise<void>;
+
   /**
    * Fetches plain rule execution events (status changes, metrics, generic events) from Event Log.
    * @throws An error if response is not OK.
@@ -33,7 +39,14 @@ export interface IRuleMonitoringApiClient {
   ): Promise<GetRuleExecutionResultsResponse>;
 }
 
-export interface FetchRuleExecutionEventsArgs {
+export interface RuleMonitoringApiCallArgs {
+  /**
+   * Optional signal for cancelling the request.
+   */
+  signal?: AbortSignal;
+}
+
+export interface FetchRuleExecutionEventsArgs extends RuleMonitoringApiCallArgs {
   /**
    * Saved Object ID of the rule (`rule.id`, not static `rule.rule_id`).
    */
@@ -63,14 +76,9 @@ export interface FetchRuleExecutionEventsArgs {
    * Number of results to fetch per page.
    */
   perPage?: number;
-
-  /**
-   * Optional signal for cancelling the request.
-   */
-  signal?: AbortSignal;
 }
 
-export interface FetchRuleExecutionResultsArgs {
+export interface FetchRuleExecutionResultsArgs extends RuleMonitoringApiCallArgs {
   /**
    * Saved Object ID of the rule (`rule.id`, not static `rule.rule_id`).
    */
@@ -116,9 +124,4 @@ export interface FetchRuleExecutionResultsArgs {
    * Number of results to fetch per page.
    */
   perPage?: number;
-
-  /**
-   * Optional signal for cancelling the request.
-   */
-  signal?: AbortSignal;
 }

x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/index.ts

@@ -12,4 +12,5 @@ export * from './components/basic/indicators/execution_status_indicator';
 export * from './components/execution_events_table/execution_events_table';
 export * from './components/execution_results_table/use_execution_results';
 
+export * from './logic/detection_engine_health/use_setup_detection_engine_health_api';
 export * from './logic/execution_settings/use_execution_settings';

x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/logic/detection_engine_health/use_setup_detection_engine_health_api.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { UseMutationOptions } from '@tanstack/react-query';
+import { useMutation } from '@tanstack/react-query';
+import { useEffect } from 'react';
+
+import { SETUP_HEALTH_URL } from '../../../../../common/detection_engine/rule_monitoring';
+import { api } from '../../api';
+
+export const SETUP_DETECTION_ENGINE_HEALTH_API_MUTATION_KEY = ['POST', SETUP_HEALTH_URL];
+
+export const useSetupDetectionEngineHealthApi = (options?: UseMutationOptions<void, Error>) => {
+  const { mutate: setupDetectionEngineHealthApi } = useMutation(
+    () => api.setupDetectionEngineHealthApi(),
+    {
+      ...options,
+      mutationKey: SETUP_DETECTION_ENGINE_HEALTH_API_MUTATION_KEY,
+    }
+  );
+
+  useEffect(() => {
+    setupDetectionEngineHealthApi();
+  }, [setupDetectionEngineHealthApi]);
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/setup/setup_health_route.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { transformError } from '@kbn/securitysolution-es-utils';
+import { buildSiemResponse } from '../../../../routes/utils';
+import type { SecuritySolutionPluginRouter } from '../../../../../../types';
+
+import { SETUP_HEALTH_URL } from '../../../../../../../common/detection_engine/rule_monitoring';
+
+/**
+ * Similar to the "setup" command of beats, this endpoint installs resources
+ * (dashboards, data views, etc) related to rule monitoring and Detection Engine health,
+ * and can do any other setup work.
+ */
+export const setupHealthRoute = (router: SecuritySolutionPluginRouter) => {
+  router.post(
+    {
+      path: SETUP_HEALTH_URL,
+      validate: {},
+      options: {
+        tags: ['access:securitySolution'],
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+
+      try {
+        const ctx = await context.resolve(['securitySolution']);
+        const healthClient = ctx.securitySolution.getDetectionEngineHealthClient();
+
+        await healthClient.installAssetsForMonitoringHealth();
+
+        return response.ok({ body: {} });
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/register_routes.ts

@@ -9,6 +9,7 @@ import type { SecuritySolutionPluginRouter } from '../../../../types';
 import { getClusterHealthRoute } from './detection_engine_health/get_cluster_health/get_cluster_health_route';
 import { getRuleHealthRoute } from './detection_engine_health/get_rule_health/get_rule_health_route';
 import { getSpaceHealthRoute } from './detection_engine_health/get_space_health/get_space_health_route';
+import { setupHealthRoute } from './detection_engine_health/setup/setup_health_route';
 import { getRuleExecutionEventsRoute } from './rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route';
 import { getRuleExecutionResultsRoute } from './rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route';
 
@@ -17,6 +18,7 @@ export const registerRuleMonitoringRoutes = (router: SecuritySolutionPluginRoute
   getClusterHealthRoute(router);
   getSpaceHealthRoute(router);
   getRuleHealthRoute(router);
+  setupHealthRoute(router);
 
   // Rule execution logs API
   getRuleExecutionEventsRoute(router);

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/__mocks__/index.ts

@@ -16,6 +16,8 @@ import type { IDetectionEngineHealthClient } from '../detection_engine_health_cl
 type CalculateRuleHealth = IDetectionEngineHealthClient['calculateRuleHealth'];
 type CalculateSpaceHealth = IDetectionEngineHealthClient['calculateSpaceHealth'];
 type CalculateClusterHealth = IDetectionEngineHealthClient['calculateClusterHealth'];
+type InstallAssetsForMonitoringHealth =
+  IDetectionEngineHealthClient['installAssetsForMonitoringHealth'];
 
 export const detectionEngineHealthClientMock = {
   create: (): jest.Mocked<IDetectionEngineHealthClient> => ({
@@ -30,5 +32,12 @@ export const detectionEngineHealthClientMock = {
     calculateClusterHealth: jest
       .fn<ReturnType<CalculateClusterHealth>, Parameters<CalculateClusterHealth>>()
       .mockResolvedValue(clusterHealthSnapshotMock.getEmptyClusterHealthSnapshot()),
+
+    installAssetsForMonitoringHealth: jest
+      .fn<
+        ReturnType<InstallAssetsForMonitoringHealth>,
+        Parameters<InstallAssetsForMonitoringHealth>
+      >()
+      .mockResolvedValue(),
   }),
 };

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/assets/data_view_kibana_event_log.json

@@ -0,0 +1,14 @@
+{
+  "type": "index-pattern",
+  "id": "kibana-event-log-data-view",
+  "managed": true,
+  "coreMigrationVersion": "8.8.0",
+  "typeMigrationVersion": "8.0.0",
+  "attributes": {
+    "name": ".kibana-event-log-*",
+    "title": ".kibana-event-log-*",
+    "timeFieldName": "@timestamp",
+    "allowNoIndex": true
+  },
+  "references": []
+}

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/assets/install_assets_for_rule_monitoring.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ISavedObjectsImporter, Logger } from '@kbn/core/server';
+import { SavedObjectsUtils } from '@kbn/core/server';
+import { cloneDeep } from 'lodash';
+import pRetry from 'p-retry';
+import { Readable } from 'stream';
+
+import sourceRuleMonitoringDashboard from './dashboard_rule_monitoring.json';
+import sourceKibanaEventLogDataView from './data_view_kibana_event_log.json';
+import sourceManagedTag from './tag_managed.json';
+import sourceSecuritySolutionTag from './tag_security_solution.json';
+
+const MAX_RETRIES = 2;
+
+/**
+ * Installs managed assets for monitoring rules and health of Detection Engine.
+ */
+export const installAssetsForRuleMonitoring = async (
+  savedObjectsImporter: ISavedObjectsImporter,
+  logger: Logger,
+  currentSpaceId: string
+): Promise<void> => {
+  const operation = async (attemptCount: number) => {
+    logger.debug(`Installing assets for rule monitoring (attempt ${attemptCount})...`);
+
+    const assets = getAssetsForRuleMonitoring(currentSpaceId);
+
+    // The assets are marked as "managed: true" at the saved object level, which in the future
+    // should be reflected in the UI for the user. Ticket to track:
+    // /~https://github.com/elastic/kibana/issues/140364
+    const importResult = await savedObjectsImporter.import({
+      readStream: Readable.from(assets),
+      managed: true,
+      overwrite: true,
+      createNewCopies: false,
+      refresh: false,
+      namespace: spaceIdToNamespace(currentSpaceId),
+    });
+
+    importResult.warnings.forEach((w) => {
+      logger.warn(w.message);
+    });
+
+    if (!importResult.success) {
+      const errors = (importResult.errors ?? []).map(
+        (e) => `Couldn't import "${e.type}:${e.id}": ${JSON.stringify(e.error)}`
+      );
+
+      errors.forEach((e) => {
+        logger.error(e);
+      });
+
+      // This will retry the operation
+      throw new Error(errors.length > 0 ? errors[0] : `Unknown error (attempt ${attemptCount})`);
+    }
+
+    logger.debug('Assets for rule monitoring installed');
+  };
+
+  await pRetry(operation, { retries: MAX_RETRIES });
+};
+
+const getAssetsForRuleMonitoring = (currentSpaceId: string) => {
+  const withSpaceId = appendSpaceId(currentSpaceId);
+
+  const assetRuleMonitoringDashboard = cloneDeep(sourceRuleMonitoringDashboard);
+  const assetKibanaEventLogDataView = cloneDeep(sourceKibanaEventLogDataView);
+  const assetManagedTag = cloneDeep(sourceManagedTag);
+  const assetSecuritySolutionTag = cloneDeep(sourceSecuritySolutionTag);
+
+  // Update ids of the assets to include the current space id
+  assetRuleMonitoringDashboard.id = withSpaceId('security-detection-rule-monitoring');
+  assetManagedTag.id = withSpaceId('fleet-managed');
+  assetSecuritySolutionTag.id = withSpaceId('security-solution');
+
+  // Update saved object references of the dashboard accordingly
+  assetRuleMonitoringDashboard.references = assetRuleMonitoringDashboard.references.map(
+    (reference) => {
+      if (reference.id === 'fleet-managed-<spaceId>') {
+        return { ...reference, id: assetManagedTag.id };
+      }
+      if (reference.id === 'security-solution-<spaceId>') {
+        return { ...reference, id: assetSecuritySolutionTag.id };
+      }
+
+      return reference;
+    }
+  );
+
+  return [
+    assetManagedTag,
+    assetSecuritySolutionTag,
+    assetKibanaEventLogDataView,
+    assetRuleMonitoringDashboard,
+  ];
+};
+
+const appendSpaceId = (spaceId: string) => (str: string) => `${str}-${spaceId}`;
+
+const spaceIdToNamespace = SavedObjectsUtils.namespaceStringToId;