From 0f0a64b5c477b75e627ccc630591eac29ca75509 Mon Sep 17 00:00:00 2001 From: Noah Masur <7386960+nmasur@users.noreply.github.com> Date: Sun, 16 Jul 2023 13:50:58 +0000 Subject: [PATCH] add victoriametrics --- hosts/swan/default.nix | 3 +- modules/nixos/hardware/zfs.nix | 5 +- modules/nixos/services/default.nix | 1 + modules/nixos/services/grafana.nix | 7 +- modules/nixos/services/nextcloud.nix | 22 +++-- modules/nixos/services/prometheus.nix | 54 +++++++----- modules/nixos/services/secrets.nix | 13 ++- modules/nixos/services/victoriametrics.nix | 95 ++++++++++++++++++++++ 8 files changed, 164 insertions(+), 36 deletions(-) create mode 100644 modules/nixos/services/victoriametrics.nix diff --git a/hosts/swan/default.nix b/hosts/swan/default.nix index 2abdd6c..d175696 100644 --- a/hosts/swan/default.nix +++ b/hosts/swan/default.nix @@ -57,7 +57,8 @@ inputs.nixpkgs.lib.nixosSystem { services.nextcloud.enable = true; services.calibre-web.enable = true; services.openssh.enable = true; - services.prometheus.enable = true; + services.prometheus.enable = false; + services.vmagent.enable = true; services.samba.enable = true; cloudflareTunnel = { diff --git a/modules/nixos/hardware/zfs.nix b/modules/nixos/hardware/zfs.nix index 6db46d4..e250d20 100644 --- a/modules/nixos/hardware/zfs.nix +++ b/modules/nixos/hardware/zfs.nix @@ -10,8 +10,9 @@ config.boot.zfs.package.latestCompatibleLinuxPackages; boot.kernelParams = [ "nohibernate" ]; boot.supportedFilesystems = [ "zfs" ]; - services.prometheus.exporters.zfs.enable = true; - scrapeTargets = [ + services.prometheus.exporters.zfs.enable = + config.prometheus.exporters.enable; + prometheus.scrapeTargets = [ "127.0.0.1:${ builtins.toString config.services.prometheus.exporters.zfs.port }" diff --git a/modules/nixos/services/default.nix b/modules/nixos/services/default.nix index a64590f..a158c6a 100644 --- a/modules/nixos/services/default.nix +++ b/modules/nixos/services/default.nix @@ -24,6 +24,7 @@ ./sshd.nix ./transmission.nix ./vaultwarden.nix + ./victoriametrics.nix ./wireguard.nix ]; diff --git a/modules/nixos/services/grafana.nix b/modules/nixos/services/grafana.nix index dc241c9..14c3b62 100644 --- a/modules/nixos/services/grafana.nix +++ b/modules/nixos/services/grafana.nix @@ -13,7 +13,12 @@ match = [{ host = [ config.hostnames.metrics ]; }]; handle = [{ handler = "reverse_proxy"; - upstreams = [{ dial = "localhost:3000"; }]; + upstreams = [{ + dial = "localhost:${ + builtins.toString + config.services.grafana.settings.server.http_port + }"; + }]; }]; }]; diff --git a/modules/nixos/services/nextcloud.nix b/modules/nixos/services/nextcloud.nix index fede4c8..39cc5a6 100644 --- a/modules/nixos/services/nextcloud.nix +++ b/modules/nixos/services/nextcloud.nix @@ -1,4 +1,10 @@ -{ config, pkgs, lib, ... }: { +{ config, pkgs, lib, ... }: + +let + + port = 8080; + +in { config = lib.mkIf config.services.nextcloud.enable { @@ -18,7 +24,7 @@ # Don't let Nginx use main ports (using Caddy instead) services.nginx.virtualHosts."localhost".listen = [{ addr = "127.0.0.1"; - port = 8080; + port = port; }]; # Point Caddy to Nginx @@ -26,7 +32,7 @@ match = [{ host = [ config.hostnames.content ]; }]; handle = [{ handler = "reverse_proxy"; - upstreams = [{ dial = "localhost:8080"; }]; + upstreams = [{ dial = "localhost:${builtins.toString port}"; }]; }]; }]; @@ -77,18 +83,20 @@ # Log metrics to prometheus services.prometheus.exporters.nextcloud = { - enable = true; + enable = config.prometheus.exporters.enable; username = config.services.nextcloud.config.adminuser; - url = "http://localhost:8080"; + url = "http://localhost:${builtins.toString port}"; passwordFile = config.services.nextcloud.config.adminpassFile; }; - scrapeTargets = [ + prometheus.scrapeTargets = [ "127.0.0.1:${ builtins.toString config.services.prometheus.exporters.nextcloud.port }" ]; # Allows nextcloud-exporter to read passwordFile - users.users.nextcloud-exporter.extraGroups = [ "nextcloud" ]; + users.users.nextcloud-exporter.extraGroups = + lib.mkIf config.services.prometheus.exporters.nextcloud.enable + [ "nextcloud" ]; }; diff --git a/modules/nixos/services/prometheus.nix b/modules/nixos/services/prometheus.nix index c477fd2..98737e6 100644 --- a/modules/nixos/services/prometheus.nix +++ b/modules/nixos/services/prometheus.nix @@ -1,9 +1,12 @@ { config, pkgs, lib, ... }: { - options.scrapeTargets = lib.mkOption { - type = lib.types.listOf lib.types.str; - description = "Prometheus scrape targets"; - default = [ ]; + options.prometheus = { + exporters.enable = lib.mkEnableOption "Enable Prometheus exporters"; + scrapeTargets = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "Prometheus scrape targets"; + default = [ ]; + }; }; config = let @@ -12,9 +15,16 @@ # not hosting Grafana, send remote Prometheus writes to primary host. isServer = config.services.grafana.enable; - in lib.mkIf config.services.prometheus.enable { + in { - scrapeTargets = [ + # Turn on exporters if any Prometheus scraper is running + prometheus.exporters.enable = builtins.any (x: x) [ + config.services.prometheus.enable + config.services.victoriametrics.enable + config.services.vmagent.enable + ]; + + prometheus.scrapeTargets = [ "127.0.0.1:${ builtins.toString config.services.prometheus.exporters.node.port }" @@ -27,9 +37,9 @@ ]; services.prometheus = { - exporters.node.enable = true; - exporters.systemd.enable = true; - exporters.process.enable = true; + exporters.node.enable = config.prometheus.exporters.enable; + exporters.systemd.enable = config.prometheus.exporters.enable; + exporters.process.enable = config.prometheus.exporters.enable; exporters.process.settings.process_names = [ # Remove nix store path from process name { @@ -66,19 +76,21 @@ }; # Create credentials file for remote Prometheus push - secrets.prometheus = lib.mkIf (!isServer) { - source = ../../../private/prometheus.age; - dest = "${config.secretsDirectory}/prometheus"; - owner = "prometheus"; - group = "prometheus"; - permissions = "0440"; - }; - systemd.services.prometheus-secret = lib.mkIf (!isServer) { - requiredBy = [ "prometheus.service" ]; - before = [ "prometheus.service" ]; - }; + secrets.prometheus = + lib.mkIf (config.services.prometheus.enable && !isServer) { + source = ../../../private/prometheus.age; + dest = "${config.secretsDirectory}/prometheus"; + owner = "prometheus"; + group = "prometheus"; + permissions = "0440"; + }; + systemd.services.prometheus-secret = + lib.mkIf (config.services.prometheus.enable && !isServer) { + requiredBy = [ "prometheus.service" ]; + before = [ "prometheus.service" ]; + }; - caddy.routes = lib.mkIf isServer [{ + caddy.routes = lib.mkIf (config.services.prometheus.enable && isServer) [{ match = [{ host = [ config.hostnames.prometheus ]; }]; handle = [{ handler = "reverse_proxy"; diff --git a/modules/nixos/services/secrets.nix b/modules/nixos/services/secrets.nix index 1a171b4..5f659b3 100644 --- a/modules/nixos/services/secrets.nix +++ b/modules/nixos/services/secrets.nix @@ -39,6 +39,11 @@ type = lib.types.str; description = "Permissions expressed as octal."; }; + prefix = lib.mkOption { + default = ""; + type = lib.types.str; + description = "Prefix for secret value (for environment files)."; + }; }; }); description = "Set of secrets to decrypt to disk."; @@ -65,10 +70,10 @@ wantedBy = [ "multi-user.target" ]; serviceConfig.Type = "oneshot"; script = '' - ${pkgs.age}/bin/age --decrypt \ - --identity ${config.identityFile} \ - --output ${attrs.dest} \ - ${attrs.source} + echo "${attrs.prefix}$( + ${pkgs.age}/bin/age --decrypt \ + --identity ${config.identityFile} ${attrs.source} + )" > ${attrs.dest} chown '${attrs.owner}':'${attrs.group}' '${attrs.dest}' chmod '${attrs.permissions}' '${attrs.dest}' diff --git a/modules/nixos/services/victoriametrics.nix b/modules/nixos/services/victoriametrics.nix new file mode 100644 index 0000000..6e0975e --- /dev/null +++ b/modules/nixos/services/victoriametrics.nix @@ -0,0 +1,95 @@ +{ config, pkgs, lib, ... }: + +let + + username = "prometheus"; + + prometheusConfig = (pkgs.formats.yaml { }).generate "prometheus.yml" { + scrape_configs = [{ + job_name = config.networking.hostName; + stream_parse = true; + static_configs = [{ targets = config.prometheus.scrapeTargets; }]; + }]; + }; + + authConfig = (pkgs.formats.yaml { }).generate "auth.yml" { + users = [{ + username = username; + password = "%{PASSWORD}"; + url_prefix = + "http://localhost${config.services.victoriametrics.listenAddress}"; + }]; + }; + + authPort = "8427"; + +in { + + config = { + + services.victoriametrics.extraOptions = + [ "-promscrape.config=${prometheusConfig}" ]; + + systemd.services.vmauth = lib.mkIf config.services.victoriametrics.enable { + description = "VictoriaMetrics basic auth proxy"; + after = [ "network.target" ]; + startLimitBurst = 5; + serviceConfig = { + Restart = "on-failure"; + RestartSec = 1; + DynamicUser = true; + EnvironmentFile = config.secrets.vmauth.dest; + ExecStart = '' + ${pkgs.victoriametrics}/bin/vmauth \ + -auth.config=${authConfig} \ + -httpListenAddr=:${authPort}''; + }; + wantedBy = [ "multi-user.target" ]; + }; + + secrets.vmauth = lib.mkIf config.services.victoriametrics.enable { + source = ../../../private/prometheus.age; + dest = "${config.secretsDirectory}/vmauth"; + prefix = "PASSWORD="; + }; + systemd.services.vmauth-secret = + lib.mkIf config.services.victoriametrics.enable { + requiredBy = [ "vmauth.service" ]; + before = [ "vmauth.service" ]; + }; + + caddy.routes = lib.mkIf config.services.victoriametrics.enable [{ + match = [{ host = [ config.hostnames.prometheus ]; }]; + handle = [{ + handler = "reverse_proxy"; + upstreams = [{ dial = "localhost:${authPort}"; }]; + }]; + }]; + + # VMAgent + + services.vmagent.prometheusConfig = prometheusConfig; # Overwritten below + systemd.services.vmagent.serviceConfig = + lib.mkIf config.services.vmagent.enable { + ExecStart = lib.mkForce '' + ${pkgs.victoriametrics}/bin/vmagent \ + -promscrape.config=${prometheusConfig} \ + -remoteWrite.url="https://${config.hostnames.prometheus}/api/v1/write" \ + -remoteWrite.basicAuth.username=${username} \ + -remoteWrite.basicAuth.passwordFile=${config.secrets.vmagent.dest}''; + }; + + secrets.vmagent = lib.mkIf config.services.vmagent.enable { + source = ../../../private/prometheus.age; + dest = "${config.secretsDirectory}/vmagent"; + owner = "vmagent"; + group = "vmagent"; + }; + systemd.services.vmagent-secret = lib.mkIf config.services.vmagent.enable { + requiredBy = [ "vmagent.service" ]; + before = [ "vmagent.service" ]; + }; + + }; + +}