858 wip

package.json

@@ -31,6 +31,9 @@
     "type": "module",
     "dependencies": {
         "@breejs/later": "^4.2.0",
+        "@fastify/rate-limit": "^9.1.0",
+        "@fastify/sensible": "^5.6.0",
+        "@fastify/static": "^7.0.4",
         "@influxdata/influxdb-client": "^1.35.0",
         "@influxdata/influxdb-client-apis": "^1.35.0",
         "axios": "^1.7.4",
@@ -40,6 +43,7 @@
         "fastify-healthcheck": "^4.4.0",
         "fastify-metrics": "^11.0.0",
         "fs-extra": "^11.2.0",
+        "handlebars": "^4.7.7",
         "influx": "^5.9.3",
         "js-yaml": "^4.1.0",
         "lodash.clonedeep": "^4.5.0",

src/butler-sos.js

@@ -24,6 +24,7 @@ import { udpInitLogEventServer } from './lib/udp_handlers_log_events.js';
 import { setupAnonUsageReportTimer } from './lib/telemetry.js';
 import { setupPromClient } from './lib/prom-client.js';
 import { verifyConfigFile } from './lib/config-file-verify.js';
+import { setupConfigVisServer } from './lib/config-visualise.js';
 
 // Suppress experimental warnings
 // https://stackoverflow.com/questions/55778283/how-to-disable-warnings-when-node-is-launched-via-a-global-shell-script
@@ -279,6 +280,11 @@ async function mainScript() {
     if (globals.config.get('Butler-SOS.appNames.enableAppNameExtract') === true) {
         setupAppNamesExtractTimer();
     }
+
+    // Set up config server, if enabled
+    if (globals.config.get('Butler-SOS.configVisualisation.enable') === true) {
+        await setupConfigVisServer();
+    }
 }
 
 mainScript();

src/config/production_template.yaml

@@ -10,6 +10,13 @@ Butler-SOS:
                           # More info on whata data is collected: https://butler-sos.ptarmiganlabs.com/docs/about/telemetry/
                           # Please consider leaving this at true - it really helps future development of Butler SOS!
 
+  # Should Butler SOS start a web server that serves an obfuscated view of the Butler SOS config file?
+  configVisualisation:
+    enable: false
+    host: localhost       # Hostname or IP address where the web server will listen. Should be localhost in most cases.
+    port: 3100            # Port where the web server will listen. Change if port 3100 is already in use.
+    obfuscate: true        # Should the config file shown in the web UI be obfuscated?
+
   # Heartbeats can be used to send "I'm alive" messages to some other tool, e.g. an infrastructure monitoring tool
   # The concept is simple: The remoteURL will be called at the specified frequency. The receiving tool will then know 
   # that Butler SOS is alive.

src/lib/config-file-schema.js

@@ -4,6 +4,12 @@ export const confifgFileSchema = {
         fileLogging: 'boolean',
         logDirectory: 'string',
         anonTelemetry: 'boolean',
+        configVisualisation: {
+            enable: 'boolean',
+            host: 'string',
+            port: 'number',
+            obfuscate: 'boolean',
+        },
         heartbeat: {
             enable: 'boolean',
             remoteURL: 'string',

src/lib/config-obfuscate.js

@@ -0,0 +1,152 @@
+import globals from '../globals.js';
+
+function configObfuscate(config) {
+    try {
+        const obfuscatedConfig = { ...config };
+
+        // Keep first 10 chars of remote URL, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].heartbeat.remoteURL = obfuscatedConfig['Butler-SOS'].heartbeat.remoteURL.substring(0, 10) + '*'.repeat(10);
+
+        // Update entries in the array obfuscatedConfig['Butler-SOS'].thirdPartyToolsCredentials.newRelic
+        obfuscatedConfig['Butler-SOS'].thirdPartyToolsCredentials.newRelic = obfuscatedConfig['Butler-SOS'].thirdPartyToolsCredentials.newRelic?.map(
+            (element) => ({
+                ...element,
+                insertApiKey: element.insertApiKey.substring(0, 5) + '*'.repeat(10),
+                accountId: element.accountId.toString().substring(0, 3) + '*'.repeat(10),
+            }),
+        );
+
+        // Obfuscate Butler-SOS.iuserEvents.udpServerConfig.serverHost, keep first 3 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].userEvents.udpServerConfig.serverHost = 
+            obfuscatedConfig['Butler-SOS'].userEvents.udpServerConfig.serverHost.substring(0, 3) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.iuserEvents.sendToMQTT.postTo.everythingTopic.topic, keep first 10 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].userEvents.sendToMQTT.postTo.everythingTopic.topic = 
+            obfuscatedConfig['Butler-SOS'].userEvents.sendToMQTT.postTo.everythingTopic.topic.substring(0, 10) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.iuserEvents.sendToMQTT.postTo.sessionStartTopic.topic, keep first 10 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].userEvents.sendToMQTT.postTo.sessionStartTopic.topic =
+            obfuscatedConfig['Butler-SOS'].userEvents.sendToMQTT.postTo.sessionStartTopic.topic.substring(0, 10) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.iuserEvents.sendToMQTT.postTo.sessionStopTopic.topic, keep first 10 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].userEvents.sendToMQTT.postTo.sessionStopTopic.topic =
+            obfuscatedConfig['Butler-SOS'].userEvents.sendToMQTT.postTo.sessionStopTopic.topic.substring(0, 10) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.iuserEvents.sendToMQTT.postTo.connectionOpenTopic.topic, keep first 10 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].userEvents.sendToMQTT.postTo.connectionOpenTopic.topic =
+            obfuscatedConfig['Butler-SOS'].userEvents.sendToMQTT.postTo.connectionOpenTopic.topic.substring(0, 10) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.iuserEvents.sendToMQTT.postTo.connectionCloseTopic.topic, keep first 10 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].userEvents.sendToMQTT.postTo.connectionCloseTopic.topic =
+            obfuscatedConfig['Butler-SOS'].userEvents.sendToMQTT.postTo.connectionCloseTopic.topic.substring(0, 10) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.logEvents.udpServerConfig.serverHost, keep first 3 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].logEvents.udpServerConfig.serverHost =
+            obfuscatedConfig['Butler-SOS'].logEvents.udpServerConfig.serverHost.substring(0, 3) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.logEvents.sendToMQTT.baseTopic, keep first 10 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].logEvents.sendToMQTT.baseTopic =
+            obfuscatedConfig['Butler-SOS'].logEvents.sendToMQTT.baseTopic.substring(0, 10) + '*'.repeat(10);
+
+        // Log db - may not be present in the config in future versions of Butler SOS
+        if (obfuscatedConfig['Butler-SOS'].logdb) {
+            // Obfuscate Butler-SOS.logdb.host, keep first 3 chars, mask the rest with *
+            obfuscatedConfig['Butler-SOS'].logdb.host = obfuscatedConfig['Butler-SOS'].logdb.host.substring(0, 3) + '*'.repeat(10);
+
+            // Obfuscate Butler-SOS.logdb.qlogsReaderUser, keep first 3 chars, mask the rest with *
+            obfuscatedConfig['Butler-SOS'].logdb.qlogsReaderUser = obfuscatedConfig['Butler-SOS'].logdb.qlogsReaderUser.substring(0, 3) + '*'.repeat(10);
+
+            // Obfuscate Butler-SOS.logdb.qlogsReaderPwdd, keep first 0 chars, mask the rest with *
+            obfuscatedConfig['Butler-SOS'].logdb.qlogsReaderPwdd = '*'.repeat(10);
+        }
+
+        // Obfuscate Butler-SOS.cert.clientCert, keep first 10 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].cert.clientCert = obfuscatedConfig['Butler-SOS'].cert.clientCert.substring(0, 10) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.cert.clientCertKey, keep first 10 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].cert.clientCertKey = obfuscatedConfig['Butler-SOS'].cert.clientCertKey.substring(0, 10) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.cert.clientCertCA, keep first 10 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].cert.clientCertCA = obfuscatedConfig['Butler-SOS'].cert.clientCertCA.substring(0, 10) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.cert.clientCertPassphrase, keep first 0 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].cert.clientCertPassphrase = '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.mqttConfig.brokerHost, keep first 3 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].mqttConfig.brokerHost = obfuscatedConfig['Butler-SOS'].mqttConfig.brokerHost.substring(0, 3) + '*'.repeat(10);
+
+
+        // Obfuscate Butler-SOS.prometheus.host, keep first 3 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].prometheus.host = obfuscatedConfig['Butler-SOS'].prometheus.host.substring(0, 3) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.influxdbConfig.host, keep first 3 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].influxdbConfig.host = obfuscatedConfig['Butler-SOS'].influxdbConfig.host.substring(0, 3) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.influxdbConfig.v2Config.org, keep first 3 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].influxdbConfig.v2Config.org = obfuscatedConfig['Butler-SOS'].influxdbConfig.v2Config.org.substring(0, 3) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.influxdbConfig.v2Config.bucket, keep first 3 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].influxdbConfig.v2Config.bucket = obfuscatedConfig['Butler-SOS'].influxdbConfig.v2Config.bucket.substring(0, 3) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.influxdbConfig.v2Config.token, keep first 0 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].influxdbConfig.v2Config.token = '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.influxdbConfig.v1Config.auth.username, keep first 3 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].influxdbConfig.v1Config.auth.username = obfuscatedConfig['Butler-SOS'].influxdbConfig.v1Config.auth.username.substring(0, 3) + '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.influxdbConfig.v1Config.auth.password, keep first 0 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].influxdbConfig.v1Config.auth.password = '*'.repeat(10);
+
+        // Obfuscate Butler-SOS.appNames.hostIP, keep first 3 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].appNames.hostIP = obfuscatedConfig['Butler-SOS'].appNames.hostIP.substring(0, 3) + '*'.repeat(10);
+
+
+
+        // Obfuscate Butler-SOS.serversToMonitor.servers[].host, keep first 3 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].serversToMonitor.servers = obfuscatedConfig['Butler-SOS'].serversToMonitor.servers?.map((element) => ({
+            ...element,
+            host: element.host.substring(0, 3) + '*'.repeat(10),
+        }));
+
+        // Obfuscate Butler-SOS.serversToMonitor.servers[].logDbHost, keep first 3 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].serversToMonitor.servers = obfuscatedConfig['Butler-SOS'].serversToMonitor.servers?.map((element) => ({
+            ...element,
+            logDbHost: element.logDbHost.substring(0, 3) + '*'.repeat(10),
+        }));
+
+        // Obfuscate Butler-SOS.serversToMonitor.servers[].userSessions.host, keep first 3 chars, mask the rest with *
+        obfuscatedConfig['Butler-SOS'].serversToMonitor.servers = obfuscatedConfig['Butler-SOS'].serversToMonitor.servers?.map((element) => ({
+            ...element,
+            userSessions: element.userSessions.host.substring(0, 3) + '*'.repeat(10),
+        }));
+
+        // Obfuscate Butler-SOS.serversToMonitor.servers[].headers, keep first 5 chars, mask the rest with * 
+        // Butler-SOS.serversToMonitor.servers[].headers is an object, so we need to obfuscate each key-value pair
+        // If the array Butler-SOS.serversToMonitor.servers[].headers is empty, no obfuscation should be done
+        obfuscatedConfig['Butler-SOS'].serversToMonitor.servers = obfuscatedConfig['Butler-SOS'].serversToMonitor.servers?.map((element) => {
+            const newHeaders = {};
+
+            // Is elemnt.headers an object with more than 0 key-value pairs?
+            if (element?.headers && Object.keys(element?.headers)?.length > 0) {
+                Object.entries(element?.headers).forEach(([key, value]) => {
+                    newHeaders[key] = value.substring(0, 5) + '*'.repeat(10);
+                });
+            }
+
+            return {
+                ...element,
+                headers: newHeaders,
+            };
+        });
+
+        return obfuscatedConfig;
+    } catch (err) {
+        globals.logger.error(`CONFIG OBFUSCATE: Error obfuscating config: ${err.message}`);
+        if (err.stack) {
+            globals.logger.error(`CONFIG OBFUSCATE: ${err.stack}`);
+        }
+        throw err;
+    }
+}
+
+export default configObfuscate;

src/lib/config-visualise.js

@@ -0,0 +1,138 @@
+import Fastify from 'fastify';
+import FastifyRateLimit from '@fastify/rate-limit';
+import FastifyStatic from '@fastify/static';
+import fs from 'fs';
+import path from 'path';
+import yaml from 'js-yaml';
+import handlebars from 'handlebars';
+
+import globals from '../globals.js';
+import configObfuscate from './config-obfuscate.js';
+
+export async function setupConfigVisServer(logger, config) {
+    try {
+        // Register rate limit for API
+        // 0 means no rate limit
+
+        // This code registers the FastifyRateLimit plugin.
+        // The plugin limits the number of API requests that
+        // can be made from a given IP address within a given
+        // time window.
+
+        const configVisServer = Fastify({ logger: true });
+
+        // Set Fastify log level based on log level in Butler config file
+        const currLogLevel = globals.getLoggingLevel();
+        if (currLogLevel === 'debug' || currLogLevel === 'silly') {
+            configVisServer.log.level = 'info';
+        } else {
+            configVisServer.log.level = 'silent';
+        }
+
+        // 30 requests per minute
+        await configVisServer.register(FastifyRateLimit, {
+            max: 300,
+            timeWindow: '1 minute',
+        });
+
+        // Add custom error handler for 429 errors (rate limit exceeded)
+        configVisServer.setErrorHandler((error, request, reply) => {
+            if (error.statusCode === 429) {
+                globals.logger.warn(
+                    `CONFIG VIS: Rate limit exceeded for source IP address ${request.ip}. Method=${request.method}, endpoint=${request.url}`,
+                );
+            }
+            reply.send(error);
+        });
+
+        // This loads all plugins defined in plugins.
+        // Those should be support plugins that are reused through your application
+        await configVisServer.register(import('../plugins/sensible.js'), { options: {} });
+        await configVisServer.register(import('../plugins/support.js'), { options: {} });
+
+        // Create absolute path to the html directory
+        // dirname points to the directory where this file (app.js) is located, taking into account
+        // if the app is running as a packaged app or as a Node.js app.
+        globals.logger.verbose(`----------------2: ${globals.appBasePath}`);
+
+        // Get directory contents of dirname
+        const dirContents = fs.readdirSync(globals.appBasePath);
+        globals.logger.verbose(`CONFIG VIS: Directory contents of "${globals.appBasePath}": ${dirContents}`);
+
+
+        const htmlDir = path.resolve(globals.appBasePath, 'static/configvis');
+        globals.logger.info(`CONFIG VIS: Serving static files from ${htmlDir}`);
+
+        await configVisServer.register(FastifyStatic, {
+            root: htmlDir,
+            constraints: {}, // optional: default {}. Example: { host: 'example.com' }
+            redirect: true, // Redirect to trailing '/' when the pathname is a dir
+        });
+
+        configVisServer.get('/', async (request, reply) => {
+            // Obfuscate the config object before sending it to the client
+            // First get clean copy of the config object
+            let newConfig = JSON.parse(JSON.stringify(globals.config));
+
+            if (globals.config.get('Butler-SOS.configVisualisation.obfuscate')) {
+                // Obfuscate config file before presenting it to the user
+                // This is done to avoid leaking sensitive information
+                // to users who should not have access to it.
+                // The obfuscation is done by replacing parts of the
+                // config file with masked strings.
+                newConfig = configObfuscate(newConfig);
+            }
+
+            // Convert the (potentially obfuscated) config object to YAML format (=string)
+            const butlerConfigYaml = yaml.dump(newConfig);
+
+            // Read index.html from disk
+            // dirname points to the directory where this file (app.js) is located, taking into account
+            // if the app is running as a packaged app or as a Node.js app.
+            globals.logger.verbose(`----------------3: ${globals.appBasePath}`);
+            const filePath = path.resolve(globals.appBasePath, 'static/configvis', 'index.html');
+            const template = fs.readFileSync(filePath, 'utf8');
+
+            // Compile handlebars template
+            const compiledTemplate = handlebars.compile(template);
+
+            // Get config as HTML encoded JSON string
+            const butlerConfigJsonEncoded = JSON.stringify(newConfig);
+
+            // Render the template
+            const renderedText = compiledTemplate({ butlerConfigJsonEncoded, butlerConfigYaml });
+
+            globals.logger.debug(`CONFIG VIS: Rendered text: ${renderedText}`);
+
+            // Send reply as HTML
+            reply.code(200).header('Content-Type', 'text/html; charset=utf-8').send(renderedText);
+        });
+
+        configVisServer.listen(
+            {
+                host: globals.config.get('Butler-SOS.configVisualisation.host'),
+                port: globals.config.get('Butler-SOS.configVisualisation.port'),
+            },
+            (err, address) => {
+                if (err) {
+                    globals.logger.error(`CONFIG VIS: Could not set up config visualisation server on ${address}`);
+                    globals.logger.error(`CONFIG VIS: ${err.stack}`);
+                    configVisServer.log.error(err);
+                    process.exit(1);
+                }
+                globals.logger.info(`CONFIG VIS: Config visualisation server listening on ${address}`);
+
+                configVisServer.ready((err2) => {
+                    if (err2) throw err;
+                });
+            },
+        );        
+    } catch (err) {
+        globals.logger.error(`CONFIG VIS: Error setting up config visualisation server: ${err.message}`);
+        if (err.stack) {
+            globals.logger.error(`CONFIG VIS: ${err.stack}`);
+        }
+        throw err;
+    }
+}
+

src/plugins/README.md

@@ -0,0 +1,16 @@
+# Plugins Folder
+
+Plugins define behavior that is common to all the routes in your
+application. Authentication, caching, templates, and all the other cross
+cutting concerns should be handled by plugins placed in this folder.
+
+Files in this folder are typically defined through the
+[`fastify-plugin`](/~https://github.com/fastify/fastify-plugin) module,
+making them non-encapsulated. They can define decorators and set hooks
+that will then be used in the rest of your application.
+
+Check out:
+
+-   [The hitchhiker's guide to plugins](https://www.fastify.io/docs/latest/Plugins-Guide/)
+-   [Fastify decorators](https://www.fastify.io/docs/latest/Decorators/).
+-   [Fastify lifecycle](https://www.fastify.io/docs/latest/Lifecycle/).

src/plugins/sensible.js

@@ -0,0 +1,14 @@
+import fp from 'fastify-plugin';
+
+/**
+ * This plugins adds some utilities to handle http errors
+ *
+ * @see /~https://github.com/fastify/fastify-sensible
+ */
+// eslint-disable-next-line no-unused-vars
+export default fp(async (fastify, _opts) => {
+    // eslint-disable-next-line global-require
+    await fastify.register(import('@fastify/sensible'), {
+        errorHandler: false,
+    });
+});