add victoriametrics

This commit is contained in:
Noah Masur 2023-07-16 13:50:58 +00:00
parent edb4ec77ca
commit 0f0a64b5c4
8 changed files with 164 additions and 36 deletions

View File

@ -57,7 +57,8 @@ inputs.nixpkgs.lib.nixosSystem {
services.nextcloud.enable = true;
services.calibre-web.enable = true;
services.openssh.enable = true;
services.prometheus.enable = true;
services.prometheus.enable = false;
services.vmagent.enable = true;
services.samba.enable = true;
cloudflareTunnel = {

View File

@ -10,8 +10,9 @@
config.boot.zfs.package.latestCompatibleLinuxPackages;
boot.kernelParams = [ "nohibernate" ];
boot.supportedFilesystems = [ "zfs" ];
services.prometheus.exporters.zfs.enable = true;
scrapeTargets = [
services.prometheus.exporters.zfs.enable =
config.prometheus.exporters.enable;
prometheus.scrapeTargets = [
"127.0.0.1:${
builtins.toString config.services.prometheus.exporters.zfs.port
}"

View File

@ -24,6 +24,7 @@
./sshd.nix
./transmission.nix
./vaultwarden.nix
./victoriametrics.nix
./wireguard.nix
];

View File

@ -13,7 +13,12 @@
match = [{ host = [ config.hostnames.metrics ]; }];
handle = [{
handler = "reverse_proxy";
upstreams = [{ dial = "localhost:3000"; }];
upstreams = [{
dial = "localhost:${
builtins.toString
config.services.grafana.settings.server.http_port
}";
}];
}];
}];

View File

@ -1,4 +1,10 @@
{ config, pkgs, lib, ... }: {
{ config, pkgs, lib, ... }:
let
port = 8080;
in {
config = lib.mkIf config.services.nextcloud.enable {
@ -18,7 +24,7 @@
# Don't let Nginx use main ports (using Caddy instead)
services.nginx.virtualHosts."localhost".listen = [{
addr = "127.0.0.1";
port = 8080;
port = port;
}];
# Point Caddy to Nginx
@ -26,7 +32,7 @@
match = [{ host = [ config.hostnames.content ]; }];
handle = [{
handler = "reverse_proxy";
upstreams = [{ dial = "localhost:8080"; }];
upstreams = [{ dial = "localhost:${builtins.toString port}"; }];
}];
}];
@ -77,18 +83,20 @@
# Log metrics to prometheus
services.prometheus.exporters.nextcloud = {
enable = true;
enable = config.prometheus.exporters.enable;
username = config.services.nextcloud.config.adminuser;
url = "http://localhost:8080";
url = "http://localhost:${builtins.toString port}";
passwordFile = config.services.nextcloud.config.adminpassFile;
};
scrapeTargets = [
prometheus.scrapeTargets = [
"127.0.0.1:${
builtins.toString config.services.prometheus.exporters.nextcloud.port
}"
];
# Allows nextcloud-exporter to read passwordFile
users.users.nextcloud-exporter.extraGroups = [ "nextcloud" ];
users.users.nextcloud-exporter.extraGroups =
lib.mkIf config.services.prometheus.exporters.nextcloud.enable
[ "nextcloud" ];
};

View File

@ -1,10 +1,13 @@
{ config, pkgs, lib, ... }: {
options.scrapeTargets = lib.mkOption {
options.prometheus = {
exporters.enable = lib.mkEnableOption "Enable Prometheus exporters";
scrapeTargets = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "Prometheus scrape targets";
default = [ ];
};
};
config = let
@ -12,9 +15,16 @@
# not hosting Grafana, send remote Prometheus writes to primary host.
isServer = config.services.grafana.enable;
in lib.mkIf config.services.prometheus.enable {
in {
scrapeTargets = [
# Turn on exporters if any Prometheus scraper is running
prometheus.exporters.enable = builtins.any (x: x) [
config.services.prometheus.enable
config.services.victoriametrics.enable
config.services.vmagent.enable
];
prometheus.scrapeTargets = [
"127.0.0.1:${
builtins.toString config.services.prometheus.exporters.node.port
}"
@ -27,9 +37,9 @@
];
services.prometheus = {
exporters.node.enable = true;
exporters.systemd.enable = true;
exporters.process.enable = true;
exporters.node.enable = config.prometheus.exporters.enable;
exporters.systemd.enable = config.prometheus.exporters.enable;
exporters.process.enable = config.prometheus.exporters.enable;
exporters.process.settings.process_names = [
# Remove nix store path from process name
{
@ -66,19 +76,21 @@
};
# Create credentials file for remote Prometheus push
secrets.prometheus = lib.mkIf (!isServer) {
secrets.prometheus =
lib.mkIf (config.services.prometheus.enable && !isServer) {
source = ../../../private/prometheus.age;
dest = "${config.secretsDirectory}/prometheus";
owner = "prometheus";
group = "prometheus";
permissions = "0440";
};
systemd.services.prometheus-secret = lib.mkIf (!isServer) {
systemd.services.prometheus-secret =
lib.mkIf (config.services.prometheus.enable && !isServer) {
requiredBy = [ "prometheus.service" ];
before = [ "prometheus.service" ];
};
caddy.routes = lib.mkIf isServer [{
caddy.routes = lib.mkIf (config.services.prometheus.enable && isServer) [{
match = [{ host = [ config.hostnames.prometheus ]; }];
handle = [{
handler = "reverse_proxy";

View File

@ -39,6 +39,11 @@
type = lib.types.str;
description = "Permissions expressed as octal.";
};
prefix = lib.mkOption {
default = "";
type = lib.types.str;
description = "Prefix for secret value (for environment files).";
};
};
});
description = "Set of secrets to decrypt to disk.";
@ -65,10 +70,10 @@
wantedBy = [ "multi-user.target" ];
serviceConfig.Type = "oneshot";
script = ''
echo "${attrs.prefix}$(
${pkgs.age}/bin/age --decrypt \
--identity ${config.identityFile} \
--output ${attrs.dest} \
${attrs.source}
--identity ${config.identityFile} ${attrs.source}
)" > ${attrs.dest}
chown '${attrs.owner}':'${attrs.group}' '${attrs.dest}'
chmod '${attrs.permissions}' '${attrs.dest}'

View File

@ -0,0 +1,95 @@
{ config, pkgs, lib, ... }:
let
username = "prometheus";
prometheusConfig = (pkgs.formats.yaml { }).generate "prometheus.yml" {
scrape_configs = [{
job_name = config.networking.hostName;
stream_parse = true;
static_configs = [{ targets = config.prometheus.scrapeTargets; }];
}];
};
authConfig = (pkgs.formats.yaml { }).generate "auth.yml" {
users = [{
username = username;
password = "%{PASSWORD}";
url_prefix =
"http://localhost${config.services.victoriametrics.listenAddress}";
}];
};
authPort = "8427";
in {
config = {
services.victoriametrics.extraOptions =
[ "-promscrape.config=${prometheusConfig}" ];
systemd.services.vmauth = lib.mkIf config.services.victoriametrics.enable {
description = "VictoriaMetrics basic auth proxy";
after = [ "network.target" ];
startLimitBurst = 5;
serviceConfig = {
Restart = "on-failure";
RestartSec = 1;
DynamicUser = true;
EnvironmentFile = config.secrets.vmauth.dest;
ExecStart = ''
${pkgs.victoriametrics}/bin/vmauth \
-auth.config=${authConfig} \
-httpListenAddr=:${authPort}'';
};
wantedBy = [ "multi-user.target" ];
};
secrets.vmauth = lib.mkIf config.services.victoriametrics.enable {
source = ../../../private/prometheus.age;
dest = "${config.secretsDirectory}/vmauth";
prefix = "PASSWORD=";
};
systemd.services.vmauth-secret =
lib.mkIf config.services.victoriametrics.enable {
requiredBy = [ "vmauth.service" ];
before = [ "vmauth.service" ];
};
caddy.routes = lib.mkIf config.services.victoriametrics.enable [{
match = [{ host = [ config.hostnames.prometheus ]; }];
handle = [{
handler = "reverse_proxy";
upstreams = [{ dial = "localhost:${authPort}"; }];
}];
}];
# VMAgent
services.vmagent.prometheusConfig = prometheusConfig; # Overwritten below
systemd.services.vmagent.serviceConfig =
lib.mkIf config.services.vmagent.enable {
ExecStart = lib.mkForce ''
${pkgs.victoriametrics}/bin/vmagent \
-promscrape.config=${prometheusConfig} \
-remoteWrite.url="https://${config.hostnames.prometheus}/api/v1/write" \
-remoteWrite.basicAuth.username=${username} \
-remoteWrite.basicAuth.passwordFile=${config.secrets.vmagent.dest}'';
};
secrets.vmagent = lib.mkIf config.services.vmagent.enable {
source = ../../../private/prometheus.age;
dest = "${config.secretsDirectory}/vmagent";
owner = "vmagent";
group = "vmagent";
};
systemd.services.vmagent-secret = lib.mkIf config.services.vmagent.enable {
requiredBy = [ "vmagent.service" ];
before = [ "vmagent.service" ];
};
};
}