add victoriametrics to tempest

requires openssh, but removing public key

flake.nix

@@ -110,20 +110,10 @@
 
   };
 
-  outputs = { self, nixpkgs, ... }@inputs:
+  outputs = { nixpkgs, ... }@inputs:
 
     let
 
-      # Common overlays to always use
-      overlays = [
-        inputs.nur.overlay
-        inputs.nix2vim.overlay
-        (import ./overlays/neovim-plugins.nix inputs)
-        (import ./overlays/calibre-web.nix)
-        (import ./overlays/disko.nix inputs)
-        (import ./overlays/tree-sitter.nix inputs)
-      ];
-
       # Global configuration for my systems
       globals = let baseName = "masu.rs";
       in rec {
@@ -135,7 +125,6 @@
         mail.imapHost = "imap.purelymail.com";
         mail.smtpHost = "smtp.purelymail.com";
         dotfilesRepo = "git@github.com:nmasur/dotfiles";
-        nixpkgs.overlays = overlays;
         hostnames = {
           git = "git.${baseName}";
           metrics = "metrics.${baseName}";
@@ -148,6 +137,16 @@
         };
       };
 
+      # Common overlays to always use
+      overlays = [
+        inputs.nur.overlay
+        inputs.nix2vim.overlay
+        (import ./overlays/neovim-plugins.nix inputs)
+        (import ./overlays/calibre-web.nix)
+        (import ./overlays/disko.nix inputs)
+        (import ./overlays/tree-sitter.nix inputs)
+      ];
+
       # System types to support.
       supportedSystems =
         [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ];
@@ -157,26 +156,20 @@
 
     in rec {
 
-      nixosModules = {
-        globals = { config }: { config = globals; };
-        common = import ./modules/common;
-        nixos = import ./modules/nixos;
-        darwin = import ./modules/darwin;
-      };
-
       # Contains my full system builds, including home-manager
       # nixos-rebuild switch --flake .#tempest
       nixosConfigurations = {
-        tempest = import ./hosts/tempest { inherit self; };
-        hydra = import ./hosts/hydra { inherit self; };
-        flame = import ./hosts/flame { inherit self; };
-        swan = import ./hosts/swan { inherit self; };
+        tempest = import ./hosts/tempest { inherit inputs globals overlays; };
+        hydra = import ./hosts/hydra { inherit inputs globals overlays; };
+        flame = import ./hosts/flame { inherit inputs globals overlays; };
+        swan = import ./hosts/swan { inherit inputs globals overlays; };
       };
 
       # Contains my full Mac system builds, including home-manager
       # darwin-rebuild switch --flake .#lookingglass
       darwinConfigurations = {
-        lookingglass = import ./hosts/lookingglass { inherit self; };
+        lookingglass =
+          import ./hosts/lookingglass { inherit inputs globals overlays; };
       };
 
       # For quickly applying home-manager settings with:
@@ -192,8 +185,10 @@
       diskoConfigurations = { root = import ./disks/root.nix; };
 
       packages = let
-        aws = system: import ./hosts/aws { inherit self system; };
-        staff = system: import ./hosts/staff { inherit self system; };
+        aws = system:
+          import ./hosts/aws { inherit inputs globals overlays system; };
+        staff = system:
+          import ./hosts/staff { inherit inputs globals overlays system; };
         neovim = system:
           let pkgs = import nixpkgs { inherit system overlays; };
           in import ./modules/common/neovim/package {

hosts/aws/default.nix

@@ -1,14 +1,17 @@
-{ self, system, ... }:
+{ inputs, system, globals, overlays, ... }:
 
-self.inputs.nixos-generators.nixosGenerate {
+inputs.nixos-generators.nixosGenerate {
   inherit system;
   format = "amazon";
   modules = [
-    self.inputs.home-manager.nixosModules.home-manager
-    self.nixosModules.globals
-    self.nixosModules.common
-    self.nixosModules.nixos
+    inputs.home-manager.nixosModules.home-manager
     {
+      nixpkgs.overlays = overlays;
+      user = globals.user;
+      fullName = globals.fullName;
+      dotfilesRepo = globals.dotfilesRepo;
+      gitName = globals.gitName;
+      gitEmail = globals.gitEmail;
       networking.hostName = "sheep";
       gui.enable = false;
       theme.colors = (import ../../colorscheme/gruvbox).dark;
@@ -18,6 +21,9 @@ self.inputs.nixos-generators.nixosGenerate {
       # AWS settings require this
       permitRootLogin = "prohibit-password";
     }
+    ../../modules/common
+    ../../modules/nixos
+    ../../modules/nixos/services/sshd.nix
   ] ++ [
     # Required to fix diskSize errors during build
     ({ ... }: { amazonImage.sizeMB = 16 * 1024; })

hosts/flame/default.nix

@@ -4,23 +4,24 @@
 # How to install:
 # https://blog.korfuri.fr/posts/2022/08/nixos-on-an-oracle-free-tier-ampere-machine/
 
-{ self, ... }:
+{ inputs, globals, overlays, ... }:
 
-self.inputs.nixpkgs.lib.nixosSystem {
+inputs.nixpkgs.lib.nixosSystem {
   system = "aarch64-linux";
   specialArgs = { };
   modules = [
-    self.inputs.home-manager.nixosModules.home-manager
-    self.nixosModules.globals
-    self.nixosModules.common
-    self.nixosModules.nixos
+    globals
+    inputs.home-manager.nixosModules.home-manager
+    ../../modules/common
+    ../../modules/nixos
     {
+      nixpkgs.overlays = overlays;
+
       # Hardware
       server = true;
       networking.hostName = "flame";
 
-      imports =
-        [ (self.inputs.nixpkgs + "/nixos/modules/profiles/qemu-guest.nix") ];
+      imports = [ (inputs.nixpkgs + "/nixos/modules/profiles/qemu-guest.nix") ];
       boot.initrd.availableKernelModules = [ "xhci_pci" "virtio_pci" "usbhid" ];
 
       fileSystems."/" = {
@@ -48,7 +49,8 @@ self.inputs.nixpkgs.lib.nixosSystem {
 
       services.caddy.enable = true;
       services.grafana.enable = true;
-      services.prometheus.enable = true;
+      services.openssh.enable = true;
+      services.victoriametrics.enable = true;
       services.gitea.enable = true;
       services.vaultwarden.enable = true;
       services.minecraft-server.enable = true; # Setup Minecraft server
@@ -70,6 +72,9 @@ self.inputs.nixpkgs.lib.nixosSystem {
         accessKeyId = "0026b0e73b2e2c80000000005";
       };
 
+      # # Grant access to Jellyfin directories from Nextcloud
+      # users.users.nextcloud.extraGroups = [ "jellyfin" ];
+
       # # Wireguard config for Transmission
       # wireguard.enable = true;
       # networking.wireguard.interfaces.wg0 = {

hosts/hydra/default.nix

@@ -1,20 +1,21 @@
 # The Hydra
 # System configuration for WSL
 
-{ self, ... }:
+{ inputs, globals, overlays, ... }:
 
-self.inputs.nixpkgs.lib.nixosSystem {
+inputs.nixpkgs.lib.nixosSystem {
   system = "x86_64-linux";
   specialArgs = { };
   modules = [
-    self.inputs.wsl.nixosModules.wsl
-    self.inputs.home-manager.nixosModules.home-manager
-    self.nixosModules.globals
-    self.nixosModules.common
-    self.nixosModules.nixos
-    self.nixosModules.wsl
+    ../../modules/common
+    ../../modules/nixos
+    ../../modules/wsl
+    globals
+    inputs.wsl.nixosModules.wsl
+    inputs.home-manager.nixosModules.home-manager
     {
       networking.hostName = "hydra";
+      nixpkgs.overlays = overlays;
       identityFile = "/home/${globals.user}/.ssh/id_ed25519";
       gui.enable = false;
       theme = {

hosts/lookingglass/default.nix

@@ -1,46 +1,46 @@
 # The Looking Glass
 # System configuration for my work Macbook
 
-{ self, ... }:
+{ inputs, globals, overlays, ... }:
 
-self.inputs.darwin.lib.darwinSystem {
+inputs.darwin.lib.darwinSystem {
   system = "x86_64-darwin";
   specialArgs = { };
   modules = [
-    self.inputs.home-manager.darwinModules.home-manager
-    self.nixosModules.common
-    self.nixosModules.darwin
-    ({ config, lib, ... }: {
-      config = rec {
-        user = lib.mkForce "Noah.Masur";
-        gitName = lib.mkForce "Noah-Masur_1701";
-        gitEmail = lib.mkForce "${user}@take2games.com";
-        nixpkgs.overlays = [ self.inputs.firefox-darwin.overlay ];
-        networking.hostName = "lookingglass";
-        identityFile = "/Users/${user}/.ssh/id_ed25519";
-        gui.enable = true;
-        theme = {
-          colors = (import ../../colorscheme/gruvbox-dark).dark;
-          dark = true;
-        };
-        mail.user = globals.user;
-        charm.enable = true;
-        neovim.enable = true;
-        mail.enable = true;
-        mail.aerc.enable = true;
-        mail.himalaya.enable = false;
-        kitty.enable = true;
-        discord.enable = true;
-        firefox.enable = true;
-        dotfiles.enable = true;
-        nixlang.enable = true;
-        terraform.enable = true;
-        python.enable = true;
-        lua.enable = true;
-        kubernetes.enable = true;
-        _1password.enable = true;
-        slack.enable = true;
-      };
+    ../../modules/common
+    ../../modules/darwin
+    (globals // rec {
+      user = "Noah.Masur";
+      gitName = "Noah-Masur_1701";
+      gitEmail = "${user}@take2games.com";
     })
+    inputs.home-manager.darwinModules.home-manager
+    {
+      nixpkgs.overlays = [ inputs.firefox-darwin.overlay ] ++ overlays;
+      networking.hostName = "lookingglass";
+      identityFile = "/Users/Noah.Masur/.ssh/id_ed25519";
+      gui.enable = true;
+      theme = {
+        colors = (import ../../colorscheme/gruvbox-dark).dark;
+        dark = true;
+      };
+      mail.user = globals.user;
+      charm.enable = true;
+      neovim.enable = true;
+      mail.enable = true;
+      mail.aerc.enable = true;
+      mail.himalaya.enable = false;
+      kitty.enable = true;
+      discord.enable = true;
+      firefox.enable = true;
+      dotfiles.enable = true;
+      nixlang.enable = true;
+      terraform.enable = true;
+      python.enable = true;
+      lua.enable = true;
+      kubernetes.enable = true;
+      _1password.enable = true;
+      slack.enable = true;
+    }
   ];
 }

hosts/staff/default.nix

@@ -1,32 +1,31 @@
 # The Staff
 # ISO configuration for my USB drive
 
-{ self, system, ... }:
+{ inputs, system, overlays, ... }:
 
-self.inputs.nixos-generators.nixosGenerate {
+inputs.nixos-generators.nixosGenerate {
   inherit system;
   format = "install-iso";
-  modules = [
-    self.nixosModules.global
-    self.nixosModules.common
-    self.nixosModules.nixos
-    ({ config, pkgs, ... }: {
-      networking.hostName = "staff";
-      users.extraUsers.root.openssh.authorizedKeys.keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB+AbmjGEwITk5CK9y7+Rg27Fokgj9QEjgc9wST6MA3s"
-      ];
-      services.openssh = {
-        enable = true;
-        ports = [ 22 ];
-        allowSFTP = true;
-        settings = {
-          GatewayPorts = "no";
-          X11Forwarding = false;
-          PasswordAuthentication = false;
-          PermitRootLogin = "yes";
-        };
+  modules = [{
+    nixpkgs.overlays = overlays;
+    networking.hostName = "staff";
+    users.extraUsers.root.openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB+AbmjGEwITk5CK9y7+Rg27Fokgj9QEjgc9wST6MA3s"
+    ];
+    services.openssh = {
+      enable = true;
+      ports = [ 22 ];
+      allowSFTP = true;
+      settings = {
+        GatewayPorts = "no";
+        X11Forwarding = false;
+        PasswordAuthentication = false;
+        PermitRootLogin = "yes";
       };
-      environment.systemPackages = with pkgs; [
+    };
+    environment.systemPackages =
+      let pkgs = import inputs.nixpkgs { inherit system overlays; };
+      in with pkgs; [
         git
         vim
         wget
@@ -36,10 +35,9 @@ self.inputs.nixos-generators.nixosGenerate {
           colors = (import ../../colorscheme/gruvbox).dark;
         })
       ];
-      nix.extraOptions = ''
-        experimental-features = nix-command flakes
-        warn-dirty = false
-      '';
-    })
-  ];
+    nix.extraOptions = ''
+      experimental-features = nix-command flakes
+      warn-dirty = false
+    '';
+  }];
 }

hosts/swan/default.nix

@@ -1,17 +1,17 @@
 # The Swan
 # System configuration for my home NAS server
 
-{ self, ... }:
+{ inputs, globals, overlays, ... }:
 
-self.inputs.nixpkgs.lib.nixosSystem {
+inputs.nixpkgs.lib.nixosSystem {
   system = "x86_64-linux";
   specialArgs = { };
   modules = [
-    self.inputs.home-manager.nixosModules.home-manager
-    self.inputs.disko.nixosModules.disko
-    self.nixosModules.globals
-    self.nixosModules.common
-    self.nixosModules.nixos
+    globals
+    inputs.home-manager.nixosModules.home-manager
+    inputs.disko.nixosModules.disko
+    ../../modules/common
+    ../../modules/nixos
     {
       # Hardware
       server = true;
@@ -46,6 +46,7 @@ self.inputs.nixpkgs.lib.nixosSystem {
 
       gui.enable = false;
       theme = { colors = (import ../../colorscheme/gruvbox).dark; };
+      nixpkgs.overlays = overlays;
       neovim.enable = true;
       cloudflare.enable = true;
       dotfiles.enable = true;
@@ -55,7 +56,9 @@ self.inputs.nixpkgs.lib.nixosSystem {
       services.jellyfin.enable = true;
       services.nextcloud.enable = true;
       services.calibre-web.enable = true;
-      services.prometheus.enable = true;
+      services.openssh.enable = true;
+      services.prometheus.enable = false;
+      services.vmagent.enable = true;
       services.samba.enable = true;
 
       cloudflareTunnel = {

hosts/tempest/default.nix

@@ -1,16 +1,18 @@
 # The Tempest
 # System configuration for my desktop
 
-{ self, ... }:
+{ inputs, globals, overlays, ... }:
 
-self.inputs.nixpkgs.lib.nixosSystem {
+inputs.nixpkgs.lib.nixosSystem {
   system = "x86_64-linux";
   modules = [
-    self.inputs.home-manager.nixosModules.home-manager
-    self.nixosModules.globals
-    self.nixosModules.common
-    self.nixosModules.nixos
+    globals
+    inputs.home-manager.nixosModules.home-manager
+    ../../modules/common
+    ../../modules/nixos
     {
+      nixpkgs.overlays = overlays;
+
       # Hardware
       physical = true;
       networking.hostName = "tempest";
@@ -51,7 +53,7 @@ self.inputs.nixpkgs.lib.nixosSystem {
 
       # Must be prepared ahead
       identityFile = "/home/${globals.user}/.ssh/id_ed25519";
-      passwordHash = self.inputs.nixpkgs.lib.fileContents ../../password.sha512;
+      passwordHash = inputs.nixpkgs.lib.fileContents ../../password.sha512;
 
       # Theming
       gui.enable = true;
@@ -59,8 +61,8 @@ self.inputs.nixpkgs.lib.nixosSystem {
         colors = (import ../../colorscheme/gruvbox-dark).dark;
         dark = true;
       };
-      wallpaper = "${self.inputs.wallpapers}/gruvbox/road.jpg";
-      gtk.theme.name = self.inputs.nixpkgs.lib.mkDefault "Adwaita-dark";
+      wallpaper = "${inputs.wallpapers}/gruvbox/road.jpg";
+      gtk.theme.name = inputs.nixpkgs.lib.mkDefault "Adwaita-dark";
 
       # Programs and services
       charm.enable = true;
@@ -89,7 +91,9 @@ self.inputs.nixpkgs.lib.nixosSystem {
         leagueoflegends.enable = true;
         ryujinx.enable = true;
       };
+      services.vmagent.enable = true;
 
+      services.openssh.enable = true; # Required for Cloudflare tunnel
       cloudflareTunnel = {
         enable = true;
         id = "ac133a82-31fb-480c-942a-cdbcd4c58173";

modules/nixos/hardware/zfs.nix

@@ -10,6 +10,13 @@
         config.boot.zfs.package.latestCompatibleLinuxPackages;
       boot.kernelParams = [ "nohibernate" ];
       boot.supportedFilesystems = [ "zfs" ];
+      services.prometheus.exporters.zfs.enable =
+        config.prometheus.exporters.enable;
+      prometheus.scrapeTargets = [
+        "127.0.0.1:${
+          builtins.toString config.services.prometheus.exporters.zfs.port
+        }"
+      ];
 
     };
 

modules/nixos/services/default.nix

@@ -24,6 +24,7 @@
     ./sshd.nix
     ./transmission.nix
     ./vaultwarden.nix
+    ./victoriametrics.nix
     ./wireguard.nix
   ];
 

modules/nixos/services/gitea-runner.nix

@@ -10,9 +10,9 @@
       enable = true;
       labels = [
         # Provide a Debian base with NodeJS for actions
-        "debian-latest:docker://node:18-bullseye"
+        # "debian-latest:docker://node:18-bullseye"
         # Fake the Ubuntu name, because Node provides no Ubuntu builds
-        "ubuntu-latest:docker://node:18-bullseye"
+        # "ubuntu-latest:docker://node:18-bullseye"
         # Provide native execution on the host using below packages
         "native:host"
       ];
@@ -31,6 +31,23 @@
       tokenFile = config.secrets.giteaRunnerToken.dest;
     };
 
+    secrets.giteaRunnerToken = {
+      source = ../../../private/gitea-runner-token.age; # TOKEN=xyz
+      dest = "${config.secretsDirectory}/gitea-runner-token";
+    };
+    systemd.services.giteaRunnerToken-secret = {
+      requiredBy = [
+        "gitea-runner-${
+          config.services.gitea-actions-runner.instances.${config.networking.hostName}.name
+        }.service"
+      ];
+      before = [
+        "gitea-runner-${
+          config.services.gitea-actions-runner.instances.${config.networking.hostName}.name
+        }.service"
+      ];
+    };
+
   };
 
 }

modules/nixos/services/grafana.nix

@@ -13,7 +13,12 @@
       match = [{ host = [ config.hostnames.metrics ]; }];
       handle = [{
         handler = "reverse_proxy";
-        upstreams = [{ dial = "localhost:3000"; }];
+        upstreams = [{
+          dial = "localhost:${
+              builtins.toString
+              config.services.grafana.settings.server.http_port
+            }";
+        }];
       }];
     }];
 

modules/nixos/services/nextcloud.nix

@@ -1,9 +1,15 @@
-{ config, pkgs, lib, ... }: {
+{ config, pkgs, lib, ... }:
+
+let
+
+  port = 8080;
+
+in {
 
   config = lib.mkIf config.services.nextcloud.enable {
 
     services.nextcloud = {
-      package = pkgs.nextcloud26; # Required to specify
+      package = pkgs.nextcloud27; # Required to specify
       datadir = "/data/nextcloud";
       https = true;
       hostName = "localhost";
@@ -11,13 +17,14 @@
       config = {
         adminpassFile = config.secrets.nextcloud.dest;
         extraTrustedDomains = [ config.hostnames.content ];
+        trustedProxies = [ "127.0.0.1" ];
       };
     };
 
     # Don't let Nginx use main ports (using Caddy instead)
     services.nginx.virtualHosts."localhost".listen = [{
       addr = "127.0.0.1";
-      port = 8080;
+      port = port;
     }];
 
     # Point Caddy to Nginx
@@ -25,7 +32,7 @@
       match = [{ host = [ config.hostnames.content ]; }];
       handle = [{
         handler = "reverse_proxy";
-        upstreams = [{ dial = "localhost:8080"; }];
+        upstreams = [{ dial = "localhost:${builtins.toString port}"; }];
       }];
     }];
 
@@ -74,6 +81,23 @@
       requires = [ "phpfpm-nextcloud.service" ];
     };
 
+    # Log metrics to prometheus
+    services.prometheus.exporters.nextcloud = {
+      enable = config.prometheus.exporters.enable;
+      username = config.services.nextcloud.config.adminuser;
+      url = "http://localhost:${builtins.toString port}";
+      passwordFile = config.services.nextcloud.config.adminpassFile;
+    };
+    prometheus.scrapeTargets = [
+      "127.0.0.1:${
+        builtins.toString config.services.prometheus.exporters.nextcloud.port
+      }"
+    ];
+    # Allows nextcloud-exporter to read passwordFile
+    users.users.nextcloud-exporter.extraGroups =
+      lib.mkIf config.services.prometheus.exporters.nextcloud.enable
+      [ "nextcloud" ];
+
   };
 
 }

modules/nixos/services/prometheus.nix

@@ -1,18 +1,56 @@
 { config, pkgs, lib, ... }: {
 
+  options.prometheus = {
+    exporters.enable = lib.mkEnableOption "Enable Prometheus exporters";
+    scrapeTargets = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "Prometheus scrape targets";
+      default = [ ];
+    };
+  };
+
   config = let
 
     # If hosting Grafana, host local Prometheus and listen for inbound jobs. If
     # not hosting Grafana, send remote Prometheus writes to primary host.
     isServer = config.services.grafana.enable;
 
-  in lib.mkIf config.services.prometheus.enable {
+  in {
+
+    # Turn on exporters if any Prometheus scraper is running
+    prometheus.exporters.enable = builtins.any (x: x) [
+      config.services.prometheus.enable
+      config.services.victoriametrics.enable
+      config.services.vmagent.enable
+    ];
+
+    prometheus.scrapeTargets = [
+      "127.0.0.1:${
+        builtins.toString config.services.prometheus.exporters.node.port
+      }"
+      "127.0.0.1:${
+        builtins.toString config.services.prometheus.exporters.systemd.port
+      }"
+      "127.0.0.1:${
+        builtins.toString config.services.prometheus.exporters.process.port
+      }"
+    ];
 
     services.prometheus = {
-      exporters.node.enable = true;
+      exporters.node.enable = config.prometheus.exporters.enable;
+      exporters.systemd.enable = config.prometheus.exporters.enable;
+      exporters.process.enable = config.prometheus.exporters.enable;
+      exporters.process.settings.process_names = [
+        # Remove nix store path from process name
+        {
+          name = "{{.Matches.Wrapped}} {{ .Matches.Args }}";
+          cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ];
+        }
+      ];
+      extraFlags = lib.mkIf isServer [ "--web.enable-remote-write-receiver" ];
       scrapeConfigs = [{
-        job_name = "local";
-        static_configs = [{ targets = [ "127.0.0.1:9100" ]; }];
+        job_name = config.networking.hostName;
+        static_configs = [{ targets = config.scrapeTargets; }];
       }];
       webExternalUrl =
         lib.mkIf isServer "https://${config.hostnames.prometheus}";
@@ -28,7 +66,7 @@
         });
       remoteWrite = lib.mkIf (!isServer) [{
         name = config.networking.hostName;
-        url = "https://${config.hostnames.prometheus}";
+        url = "https://${config.hostnames.prometheus}/api/v1/write";
         basic_auth = {
           # Uses password hashed with bcrypt above
           username = "prometheus";
@@ -38,23 +76,26 @@
     };
 
     # Create credentials file for remote Prometheus push
-    secrets.prometheus = lib.mkIf (!isServer) {
-      source = ../../../private/prometheus.age;
-      dest = "${config.secretsDirectory}/prometheus";
-      owner = "prometheus";
-      group = "prometheus";
-      permissions = "0440";
-    };
-    systemd.services.prometheus-secret = lib.mkIf (!isServer) {
-      requiredBy = [ "prometheus.service" ];
-      before = [ "prometheus.service" ];
-    };
+    secrets.prometheus =
+      lib.mkIf (config.services.prometheus.enable && !isServer) {
+        source = ../../../private/prometheus.age;
+        dest = "${config.secretsDirectory}/prometheus";
+        owner = "prometheus";
+        group = "prometheus";
+        permissions = "0440";
+      };
+    systemd.services.prometheus-secret =
+      lib.mkIf (config.services.prometheus.enable && !isServer) {
+        requiredBy = [ "prometheus.service" ];
+        before = [ "prometheus.service" ];
+      };
 
-    caddy.routes = lib.mkIf isServer [{
+    caddy.routes = lib.mkIf (config.services.prometheus.enable && isServer) [{
       match = [{ host = [ config.hostnames.prometheus ]; }];
       handle = [{
         handler = "reverse_proxy";
-        upstreams = [{ dial = "localhost:9090"; }];
+        upstreams =
+          [{ dial = "localhost:${config.services.prometheus.port}"; }];
       }];
     }];
 

modules/nixos/services/secrets.nix

@@ -39,6 +39,11 @@
             type = lib.types.str;
             description = "Permissions expressed as octal.";
           };
+          prefix = lib.mkOption {
+            default = "";
+            type = lib.types.str;
+            description = "Prefix for secret value (for environment files).";
+          };
         };
       });
       description = "Set of secrets to decrypt to disk.";
@@ -65,10 +70,10 @@
         wantedBy = [ "multi-user.target" ];
         serviceConfig.Type = "oneshot";
         script = ''
-          ${pkgs.age}/bin/age --decrypt \
-            --identity ${config.identityFile} \
-            --output ${attrs.dest} \
-            ${attrs.source}
+          echo "${attrs.prefix}$(
+            ${pkgs.age}/bin/age --decrypt \
+                --identity ${config.identityFile} ${attrs.source}
+          )" > ${attrs.dest}
 
           chown '${attrs.owner}':'${attrs.group}' '${attrs.dest}'
           chmod '${attrs.permissions}' '${attrs.dest}'

modules/nixos/services/sshd.nix

@@ -13,9 +13,8 @@
     };
   };
 
-  config = lib.mkIf (config.publicKey != null) {
+  config = lib.mkIf config.services.openssh.enable {
     services.openssh = {
-      enable = true;
       ports = [ 22 ];
       allowSFTP = true;
       settings = {
@@ -27,7 +26,7 @@
     };
 
     users.users.${config.user}.openssh.authorizedKeys.keys =
-      [ config.publicKey ];
+      lib.mkIf (config.publicKey != null) [ config.publicKey ];
 
     # Implement a simple fail2ban service for sshd
     services.sshguard.enable = true;

modules/nixos/services/victoriametrics.nix

@@ -0,0 +1,95 @@
+{ config, pkgs, lib, ... }:
+
+let
+
+  username = "prometheus";
+
+  prometheusConfig = (pkgs.formats.yaml { }).generate "prometheus.yml" {
+    scrape_configs = [{
+      job_name = config.networking.hostName;
+      stream_parse = true;
+      static_configs = [{ targets = config.prometheus.scrapeTargets; }];
+    }];
+  };
+
+  authConfig = (pkgs.formats.yaml { }).generate "auth.yml" {
+    users = [{
+      username = username;
+      password = "%{PASSWORD}";
+      url_prefix =
+        "http://localhost${config.services.victoriametrics.listenAddress}";
+    }];
+  };
+
+  authPort = "8427";
+
+in {
+
+  config = {
+
+    services.victoriametrics.extraOptions =
+      [ "-promscrape.config=${prometheusConfig}" ];
+
+    systemd.services.vmauth = lib.mkIf config.services.victoriametrics.enable {
+      description = "VictoriaMetrics basic auth proxy";
+      after = [ "network.target" ];
+      startLimitBurst = 5;
+      serviceConfig = {
+        Restart = "on-failure";
+        RestartSec = 1;
+        DynamicUser = true;
+        EnvironmentFile = config.secrets.vmauth.dest;
+        ExecStart = ''
+          ${pkgs.victoriametrics}/bin/vmauth \
+                    -auth.config=${authConfig} \
+                    -httpListenAddr=:${authPort}'';
+      };
+      wantedBy = [ "multi-user.target" ];
+    };
+
+    secrets.vmauth = lib.mkIf config.services.victoriametrics.enable {
+      source = ../../../private/prometheus.age;
+      dest = "${config.secretsDirectory}/vmauth";
+      prefix = "PASSWORD=";
+    };
+    systemd.services.vmauth-secret =
+      lib.mkIf config.services.victoriametrics.enable {
+        requiredBy = [ "vmauth.service" ];
+        before = [ "vmauth.service" ];
+      };
+
+    caddy.routes = lib.mkIf config.services.victoriametrics.enable [{
+      match = [{ host = [ config.hostnames.prometheus ]; }];
+      handle = [{
+        handler = "reverse_proxy";
+        upstreams = [{ dial = "localhost:${authPort}"; }];
+      }];
+    }];
+
+    # VMAgent
+
+    services.vmagent.prometheusConfig = prometheusConfig; # Overwritten below
+    systemd.services.vmagent.serviceConfig =
+      lib.mkIf config.services.vmagent.enable {
+        ExecStart = lib.mkForce ''
+          ${pkgs.victoriametrics}/bin/vmagent \
+                  -promscrape.config=${prometheusConfig} \
+                  -remoteWrite.url="https://${config.hostnames.prometheus}/api/v1/write" \
+                  -remoteWrite.basicAuth.username=${username} \
+                  -remoteWrite.basicAuth.passwordFile=${config.secrets.vmagent.dest}'';
+      };
+
+    secrets.vmagent = lib.mkIf config.services.vmagent.enable {
+      source = ../../../private/prometheus.age;
+      dest = "${config.secretsDirectory}/vmagent";
+      owner = "vmagent";
+      group = "vmagent";
+    };
+    systemd.services.vmagent-secret = lib.mkIf config.services.vmagent.enable {
+      requiredBy = [ "vmagent.service" ];
+      before = [ "vmagent.service" ];
+    };
+
+  };
+
+}

private/gitea-runner-token.age

@@ -0,0 +1,12 @@
+-----BEGIN AGE ENCRYPTED FILE-----
+YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IE1nSGFPdyBoOVF1
+NmZocHpQQnRJcWpWUHh2bU93NkdnZWNzSlFiaHdTd24rcHpsczFRCmJaSzNkNGs1
+UDJCN2dYUVE3UTE1OU5RUWljQlN4dmxuUnpOMFYxQTdUaVEKLT4gc3NoLWVkMjU1
+MTkgWXlTVU1RIE5HdGd6aTlKM0lFUlYzT1VhS05nZ2ZxTndVZHBNQlJxYlovdXkx
+ei96d2cKdzlUYVFFaEIzaS9LZmY3MzM1RmNnR0xjOEpHK1kxM0FMTWRQSlVnczVF
+dwotPiBzc2gtZWQyNTUxOSBuanZYNUEgQ1lhMGQvUy9OWkRBR3BZV1pFNmNtb2pq
+Y2VEUzhRWGVWUkZJY1l4RGtWdwphdFZtM0ZLZURvYVZQYjV4bWVPdWJxa3RmWmVh
+SHl0T0pQWmxnVlFPR2drCi0tLSBnd2lwS3dqUk5Jelg0b3RxbFdEcnJ6ZkkvZTVN
+UllBeUUyOXBxVDBKMG5BCkGo9kj9sMVhbnXVM35lGScAb8r5LH9vf5jOdhLC/Wj2
++uA0ONIh7F2GELzf5Cw1KZJ8aHTURM2r41vZvfAQN1RwrmYOiUzlyMrvTDe78cY=
+-----END AGE ENCRYPTED FILE-----