From cc5cfc41a4ae9228125c29a9154e04e448653112 Mon Sep 17 00:00:00 2001 From: dy0gu Date: Wed, 27 Nov 2024 20:31:55 +0000 Subject: [PATCH] fix: improve label names --- README.md | 55 ++++++++++++++++++++++++--------- grafana/dashboards/logs.json | 5 +-- grafana/dashboards/metrics.json | 2 +- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e99f3c3..fa7833b 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,48 @@ # Like CCTV, but for the server πŸ“ΈπŸš¨ -- If using Loki with locally running processes you should also install Promtail, since we're only using Loki to for Docker container logs we don't need Promtail. -- Install the Loki Docker driver plugin to make docker automatically send container logs to Loki: +This repository contains a base **Grafana** setup that uses **Loki** and **Prometheus** as datasources for visualization. Note that the current configuration is meant to be used on standalone _Linux_ server hosts and not in any sort of cluster node from management systems like Kubernetes or Docker Swarm. + +We try to maintain the simplity of the configuration as much as possible seeing as we don't have any need for orchestration level workflows and monitoring. + +**Loki** is a log aggregator, it's usually used with Promtail to get a log feed from running processes but since all our projects have containerization ennforced we simply use the [grafana/loki-docker-driver](https://hub.docker.com/r/grafana/loki-docker-driver) which does that for us with Docker containers. + +**Prometheus** is a time series database, that in our case is used for monitoring system metrics. Much like **Loki** it needs a data feed, which is provided by the **node-exporter** service. + +**Grafana** provides a web interface for visualizing the data from **Loki** and **Prometheus**. It also provides an alert system based on user rules that can act through SMTP, Webhooks, etc. This configurations comes **provisioned with two dashboards**, one for metrics and another for logs, complete with filters and graphs. There is also a **provisioned alert workflow** that sends an email through SMTP when the CPU, RAM or disk usage reach certain thresholds. + +## Requirements πŸ“‹ + +- **Linux** +- Docker Engine 25.0.0+ +- Docker Compose 2.24.0+ + +## Usage πŸš€ + +- Clone the repository and open a terminal **inside** it. + +- Create a `.env` file inside the `backend` directory based on the `.env.example` file. + +- Install the [grafana/loki-docker-driver](https://hub.docker.com/r/grafana/loki-docker-driver) plugin: ```shell - docker plugin install grafana/loki-docker-driver:2.9.2 --alias loki --grant-all-permissions + docker plugin install grafana/loki-docker-driver:3.2.1 --alias loki --grant-all-permissions docker plugin ls ``` -- Change the default logging driver to the newly installed plugin: +- Change the default logging driver to the newly installed plugin in the respective `daemon.json` file: ```shell nano /etc/docker/daemon.json ``` - - Edit the default Docker logs config, you must manually change the `{REPLACE}` to the `LOKI_PORT` from the `.env` file: + - Add the following content, replacing the `port` number with the one you set for **Loki** in the `.env` file: ```json { "debug" : true, "log-driver": "loki", "log-opts": { - "loki-url": "http://localhost:{REPLACE}/loki/api/v1/push", + "loki-url": "http://localhost:port/loki/api/v1/push", "loki-batch-size": "400", "loki-retries": "3", "loki-max-backoff": "800ms", @@ -30,25 +51,29 @@ } ``` -- Restart Docker to apply the changes, note that all containers not set to restart automatically will be stopped: +- Restart to apply changes: ```shell systemctl restart docker ``` -- If you ever need to update the Docker driver plugin do it like so: +- Start the docker compose: ```shell - docker plugin disable loki --force - docker plugin upgrade loki grafana/loki-docker-driver:new-version --grant-all-permissions - docker plugin enable loki - systemctl restart docker + docker compose up --force-recreate ``` -- Start the docker compose file: +## Notes πŸ“ + +- To update the [grafana/loki-docker-driver](https://hub.docker.com/r/grafana/loki-docker-driver) driver plugin: ```shell - docker compose up --force-recreate + docker plugin disable loki --force + docker plugin upgrade loki grafana/loki-docker-driver:latest --grant-all-permissions + docker plugin enable loki + systemctl restart docker ``` -- To update any of the services simply bump the version number in the `image` field and run the `docker compose up --force-recreate` command again, but beware of breaking changes! You should look at the update notes of each service to see if there are any migration steps required when upgrading. +- To update any of the docker services simply bump the versions in the `image` fields of the `docker-compose.yaml` file and run the `docker compose up --force-recreate` command to restart them. + +- Always beware of **breaking changes** when updating to the latest versions! Look at the update notes of each component to see if there are any migration steps required when upgrading. diff --git a/grafana/dashboards/logs.json b/grafana/dashboards/logs.json index e43b8bc..80bafce 100644 --- a/grafana/dashboards/logs.json +++ b/grafana/dashboards/logs.json @@ -94,7 +94,8 @@ }, "datasource": "Loki", "includeAll": true, - "label": "Container", + "label": "Containers", + "multi": true, "name": "container_name", "options": [], "query": "label_values(container_name)", @@ -106,7 +107,7 @@ "text": "", "value": "" }, - "label": "Search Query", + "label": "Search", "name": "search_query", "type": "textbox", "hide": 0, diff --git a/grafana/dashboards/metrics.json b/grafana/dashboards/metrics.json index 738e62c..bd28de0 100644 --- a/grafana/dashboards/metrics.json +++ b/grafana/dashboards/metrics.json @@ -23502,7 +23502,7 @@ "refresh": "1h", "schemaVersion": 40, "tags": [ - "server" + "host" ], "templating": { "list": [