dotfiles/modules/nixos/services/prometheus.nix

120 lines
4.1 KiB
Nix
Raw Normal View History

2024-01-10 04:11:11 +00:00
# Prometheus is a timeseries database that exposes system and service metrics
# for use in visualizing, monitoring, and alerting (with Grafana).
# Instead of running traditional Prometheus, I generally run VictoriaMetrics as
# a more efficient drop-in replacement.
{ config, pkgs, lib, ... }: {
2022-10-14 04:01:41 +00:00
2023-07-16 13:50:58 +00:00
options.prometheus = {
exporters.enable = lib.mkEnableOption "Enable Prometheus exporters";
scrapeTargets = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "Prometheus scrape targets";
default = [ ];
};
};
2023-07-05 20:19:54 +00:00
config = let
2022-10-14 04:01:41 +00:00
2023-07-05 20:19:54 +00:00
# If hosting Grafana, host local Prometheus and listen for inbound jobs. If
# not hosting Grafana, send remote Prometheus writes to primary host.
isServer = config.services.grafana.enable;
2023-07-16 13:50:58 +00:00
in {
# Turn on exporters if any Prometheus scraper is running
prometheus.exporters.enable = builtins.any (x: x) [
config.services.prometheus.enable
config.services.victoriametrics.enable
config.services.vmagent.enable
];
2023-07-16 13:50:58 +00:00
prometheus.scrapeTargets = [
"127.0.0.1:${
builtins.toString config.services.prometheus.exporters.node.port
}"
"127.0.0.1:${
builtins.toString config.services.prometheus.exporters.systemd.port
}"
"127.0.0.1:${
builtins.toString config.services.prometheus.exporters.process.port
}"
];
2022-10-14 04:01:41 +00:00
services.prometheus = {
2023-07-16 13:50:58 +00:00
exporters.node.enable = config.prometheus.exporters.enable;
2023-07-16 20:13:41 +00:00
exporters.node.enabledCollectors = [ ];
exporters.node.disabledCollectors = [ "cpufreq" ];
2023-07-16 13:50:58 +00:00
exporters.systemd.enable = config.prometheus.exporters.enable;
exporters.process.enable = config.prometheus.exporters.enable;
exporters.process.settings.process_names = [
# Remove nix store path from process name
{
name = "{{.Matches.Wrapped}} {{ .Matches.Args }}";
cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ];
}
];
2023-07-14 02:52:23 +00:00
extraFlags = lib.mkIf isServer [ "--web.enable-remote-write-receiver" ];
2022-10-14 04:01:41 +00:00
scrapeConfigs = [{
2023-07-14 02:52:23 +00:00
job_name = config.networking.hostName;
static_configs = [{ targets = config.scrapeTargets; }];
2022-10-14 04:01:41 +00:00
}];
2023-07-07 16:16:07 +00:00
webExternalUrl =
lib.mkIf isServer "https://${config.hostnames.prometheus}";
2023-07-04 23:05:56 +00:00
# Web config file: https://prometheus.io/docs/prometheus/latest/configuration/https/
2023-07-05 20:19:54 +00:00
webConfigFile = lib.mkIf isServer
2023-07-04 23:21:53 +00:00
((pkgs.formats.yaml { }).generate "webconfig.yml" {
basic_auth_users = {
# Generate password: htpasswd -nBC 10 "" | tr -d ':\n'
# Encrypt and place in private/prometheus.age
"prometheus" =
"$2y$10$r7FWHLHTGPAY312PdhkPEuvb05aGn9Nk1IO7qtUUUjmaDl35l6sLa";
};
2023-07-04 23:21:53 +00:00
});
2023-07-05 20:19:54 +00:00
remoteWrite = lib.mkIf (!isServer) [{
name = config.networking.hostName;
2023-07-14 02:52:23 +00:00
url = "https://${config.hostnames.prometheus}/api/v1/write";
basic_auth = {
# Uses password hashed with bcrypt above
username = "prometheus";
password_file = config.secrets.prometheus.dest;
};
}];
};
# Create credentials file for remote Prometheus push
2023-07-16 13:50:58 +00:00
secrets.prometheus =
lib.mkIf (config.services.prometheus.enable && !isServer) {
source = ../../../private/prometheus.age;
dest = "${config.secretsDirectory}/prometheus";
owner = "prometheus";
group = "prometheus";
permissions = "0440";
};
systemd.services.prometheus-secret =
lib.mkIf (config.services.prometheus.enable && !isServer) {
requiredBy = [ "prometheus.service" ];
before = [ "prometheus.service" ];
};
2022-10-14 04:01:41 +00:00
2023-07-16 13:50:58 +00:00
caddy.routes = lib.mkIf (config.services.prometheus.enable && isServer) [{
2023-07-07 16:16:07 +00:00
match = [{ host = [ config.hostnames.prometheus ]; }];
2022-10-14 04:01:41 +00:00
handle = [{
handler = "reverse_proxy";
2023-07-16 03:34:03 +00:00
upstreams =
[{ dial = "localhost:${config.services.prometheus.port}"; }];
2022-10-14 04:01:41 +00:00
}];
}];
# Configure Cloudflare DNS to point to this machine
services.cloudflare-dyndns.domains =
if (config.services.prometheus.enable && isServer) then
[ config.hostnames.prometheus ]
else
[ ];
2022-10-14 04:01:41 +00:00
};
}