add victoriametrics

hosts/swan/default.nix

@@ -57,7 +57,8 @@ inputs.nixpkgs.lib.nixosSystem {
       services.nextcloud.enable = true;
       services.calibre-web.enable = true;
       services.openssh.enable = true;
-      services.prometheus.enable = true;
+      services.prometheus.enable = false;
+      services.vmagent.enable = true;
       services.samba.enable = true;
 
       cloudflareTunnel = {

modules/nixos/hardware/zfs.nix

@@ -10,8 +10,9 @@
         config.boot.zfs.package.latestCompatibleLinuxPackages;
       boot.kernelParams = [ "nohibernate" ];
       boot.supportedFilesystems = [ "zfs" ];
-      services.prometheus.exporters.zfs.enable = true;
+      services.prometheus.exporters.zfs.enable =
-      scrapeTargets = [
+        config.prometheus.exporters.enable;
+      prometheus.scrapeTargets = [
         "127.0.0.1:${
           builtins.toString config.services.prometheus.exporters.zfs.port
         }"

modules/nixos/services/default.nix

@@ -24,6 +24,7 @@
     ./sshd.nix
     ./transmission.nix
     ./vaultwarden.nix
+    ./victoriametrics.nix
     ./wireguard.nix
   ];
 

modules/nixos/services/grafana.nix

@@ -13,7 +13,12 @@
       match = [{ host = [ config.hostnames.metrics ]; }];
       handle = [{
         handler = "reverse_proxy";
-        upstreams = [{ dial = "localhost:3000"; }];
+        upstreams = [{
+          dial = "localhost:${
+              builtins.toString
+              config.services.grafana.settings.server.http_port
+            }";
+        }];
       }];
     }];
 

modules/nixos/services/nextcloud.nix

@@ -1,4 +1,10 @@
-{ config, pkgs, lib, ... }: {
+{ config, pkgs, lib, ... }:
+
+let
+
+  port = 8080;
+
+in {
 
   config = lib.mkIf config.services.nextcloud.enable {
 
@@ -18,7 +24,7 @@
     # Don't let Nginx use main ports (using Caddy instead)
     services.nginx.virtualHosts."localhost".listen = [{
       addr = "127.0.0.1";
-      port = 8080;
+      port = port;
     }];
 
     # Point Caddy to Nginx
@@ -26,7 +32,7 @@
       match = [{ host = [ config.hostnames.content ]; }];
       handle = [{
         handler = "reverse_proxy";
-        upstreams = [{ dial = "localhost:8080"; }];
+        upstreams = [{ dial = "localhost:${builtins.toString port}"; }];
       }];
     }];
 
@@ -77,18 +83,20 @@
 
     # Log metrics to prometheus
     services.prometheus.exporters.nextcloud = {
-      enable = true;
+      enable = config.prometheus.exporters.enable;
       username = config.services.nextcloud.config.adminuser;
-      url = "http://localhost:8080";
+      url = "http://localhost:${builtins.toString port}";
       passwordFile = config.services.nextcloud.config.adminpassFile;
     };
-    scrapeTargets = [
+    prometheus.scrapeTargets = [
       "127.0.0.1:${
         builtins.toString config.services.prometheus.exporters.nextcloud.port
       }"
     ];
     # Allows nextcloud-exporter to read passwordFile
-    users.users.nextcloud-exporter.extraGroups = [ "nextcloud" ];
+    users.users.nextcloud-exporter.extraGroups =
+      lib.mkIf config.services.prometheus.exporters.nextcloud.enable
+      [ "nextcloud" ];
 
   };
 

modules/nixos/services/prometheus.nix

@@ -1,10 +1,13 @@
 { config, pkgs, lib, ... }: {
 
-  options.scrapeTargets = lib.mkOption {
+  options.prometheus = {
+    exporters.enable = lib.mkEnableOption "Enable Prometheus exporters";
+    scrapeTargets = lib.mkOption {
       type = lib.types.listOf lib.types.str;
       description = "Prometheus scrape targets";
       default = [ ];
     };
+  };
 
   config = let
 
@@ -12,9 +15,16 @@
     # not hosting Grafana, send remote Prometheus writes to primary host.
     isServer = config.services.grafana.enable;
 
-  in lib.mkIf config.services.prometheus.enable {
+  in {
 
-    scrapeTargets = [
+    # Turn on exporters if any Prometheus scraper is running
+    prometheus.exporters.enable = builtins.any (x: x) [
+      config.services.prometheus.enable
+      config.services.victoriametrics.enable
+      config.services.vmagent.enable
+    ];
+
+    prometheus.scrapeTargets = [
       "127.0.0.1:${
         builtins.toString config.services.prometheus.exporters.node.port
       }"
@@ -27,9 +37,9 @@
     ];
 
     services.prometheus = {
-      exporters.node.enable = true;
+      exporters.node.enable = config.prometheus.exporters.enable;
-      exporters.systemd.enable = true;
+      exporters.systemd.enable = config.prometheus.exporters.enable;
-      exporters.process.enable = true;
+      exporters.process.enable = config.prometheus.exporters.enable;
       exporters.process.settings.process_names = [
         # Remove nix store path from process name
         {
@@ -66,19 +76,21 @@
     };
 
     # Create credentials file for remote Prometheus push
-    secrets.prometheus = lib.mkIf (!isServer) {
+    secrets.prometheus =
+      lib.mkIf (config.services.prometheus.enable && !isServer) {
         source = ../../../private/prometheus.age;
         dest = "${config.secretsDirectory}/prometheus";
         owner = "prometheus";
         group = "prometheus";
         permissions = "0440";
       };
-    systemd.services.prometheus-secret = lib.mkIf (!isServer) {
+    systemd.services.prometheus-secret =
+      lib.mkIf (config.services.prometheus.enable && !isServer) {
         requiredBy = [ "prometheus.service" ];
         before = [ "prometheus.service" ];
       };
 
-    caddy.routes = lib.mkIf isServer [{
+    caddy.routes = lib.mkIf (config.services.prometheus.enable && isServer) [{
       match = [{ host = [ config.hostnames.prometheus ]; }];
       handle = [{
         handler = "reverse_proxy";

modules/nixos/services/secrets.nix

@@ -39,6 +39,11 @@
             type = lib.types.str;
             description = "Permissions expressed as octal.";
           };
+          prefix = lib.mkOption {
+            default = "";
+            type = lib.types.str;
+            description = "Prefix for secret value (for environment files).";
+          };
         };
       });
       description = "Set of secrets to decrypt to disk.";
@@ -65,10 +70,10 @@
         wantedBy = [ "multi-user.target" ];
         serviceConfig.Type = "oneshot";
         script = ''
+          echo "${attrs.prefix}$(
             ${pkgs.age}/bin/age --decrypt \
-            --identity ${config.identityFile} \
+                --identity ${config.identityFile} ${attrs.source}
-            --output ${attrs.dest} \
+          )" > ${attrs.dest}
-            ${attrs.source}
 
           chown '${attrs.owner}':'${attrs.group}' '${attrs.dest}'
           chmod '${attrs.permissions}' '${attrs.dest}'

modules/nixos/services/victoriametrics.nix

@@ -0,0 +1,95 @@
+{ config, pkgs, lib, ... }:
+
+let
+
+  username = "prometheus";
+
+  prometheusConfig = (pkgs.formats.yaml { }).generate "prometheus.yml" {
+    scrape_configs = [{
+      job_name = config.networking.hostName;
+      stream_parse = true;
+      static_configs = [{ targets = config.prometheus.scrapeTargets; }];
+    }];
+  };
+
+  authConfig = (pkgs.formats.yaml { }).generate "auth.yml" {
+    users = [{
+      username = username;
+      password = "%{PASSWORD}";
+      url_prefix =
+        "http://localhost${config.services.victoriametrics.listenAddress}";
+    }];
+  };
+
+  authPort = "8427";
+
+in {
+
+  config = {
+
+    services.victoriametrics.extraOptions =
+      [ "-promscrape.config=${prometheusConfig}" ];
+
+    systemd.services.vmauth = lib.mkIf config.services.victoriametrics.enable {
+      description = "VictoriaMetrics basic auth proxy";
+      after = [ "network.target" ];
+      startLimitBurst = 5;
+      serviceConfig = {
+        Restart = "on-failure";
+        RestartSec = 1;
+        DynamicUser = true;
+        EnvironmentFile = config.secrets.vmauth.dest;
+        ExecStart = ''
+          ${pkgs.victoriametrics}/bin/vmauth \
+                    -auth.config=${authConfig} \
+                    -httpListenAddr=:${authPort}'';
+      };
+      wantedBy = [ "multi-user.target" ];
+    };
+
+    secrets.vmauth = lib.mkIf config.services.victoriametrics.enable {
+      source = ../../../private/prometheus.age;
+      dest = "${config.secretsDirectory}/vmauth";
+      prefix = "PASSWORD=";
+    };
+    systemd.services.vmauth-secret =
+      lib.mkIf config.services.victoriametrics.enable {
+        requiredBy = [ "vmauth.service" ];
+        before = [ "vmauth.service" ];
+      };
+
+    caddy.routes = lib.mkIf config.services.victoriametrics.enable [{
+      match = [{ host = [ config.hostnames.prometheus ]; }];
+      handle = [{
+        handler = "reverse_proxy";
+        upstreams = [{ dial = "localhost:${authPort}"; }];
+      }];
+    }];
+
+    # VMAgent
+
+    services.vmagent.prometheusConfig = prometheusConfig; # Overwritten below
+    systemd.services.vmagent.serviceConfig =
+      lib.mkIf config.services.vmagent.enable {
+        ExecStart = lib.mkForce ''
+          ${pkgs.victoriametrics}/bin/vmagent \
+                  -promscrape.config=${prometheusConfig} \
+                  -remoteWrite.url="https://${config.hostnames.prometheus}/api/v1/write" \
+                  -remoteWrite.basicAuth.username=${username} \
+                  -remoteWrite.basicAuth.passwordFile=${config.secrets.vmagent.dest}'';
+      };
+
+    secrets.vmagent = lib.mkIf config.services.vmagent.enable {
+      source = ../../../private/prometheus.age;
+      dest = "${config.secretsDirectory}/vmagent";
+      owner = "vmagent";
+      group = "vmagent";
+    };
+    systemd.services.vmagent-secret = lib.mkIf config.services.vmagent.enable {
+      requiredBy = [ "vmagent.service" ];
+      before = [ "vmagent.service" ];
+    };
+
+  };
+
+}