dotfiles/modules/nixos/services/prometheus.nix

# Prometheus is a timeseries database that exposes system and service metrics
# for use in visualizing, monitoring, and alerting (with Grafana).

# Instead of running traditional Prometheus, I generally run VictoriaMetrics as
# a more efficient drop-in replacement.

{ config, pkgs, lib, ... }: {

  options.prometheus = {
    exporters.enable = lib.mkEnableOption "Enable Prometheus exporters";
    scrapeTargets = lib.mkOption {
      type = lib.types.listOf lib.types.str;
      description = "Prometheus scrape targets";
      default = [ ];
    };
  };

  config = let

    # If hosting Grafana, host local Prometheus and listen for inbound jobs. If
    # not hosting Grafana, send remote Prometheus writes to primary host.
    isServer = config.services.grafana.enable;

  in {

    # Turn on exporters if any Prometheus scraper is running
    prometheus.exporters.enable = builtins.any (x: x) [
      config.services.prometheus.enable
      config.services.victoriametrics.enable
      config.services.vmagent.enable
    ];

    prometheus.scrapeTargets = [
      "127.0.0.1:${
        builtins.toString config.services.prometheus.exporters.node.port
      }"
      "127.0.0.1:${
        builtins.toString config.services.prometheus.exporters.systemd.port
      }"
      "127.0.0.1:${
        builtins.toString config.services.prometheus.exporters.process.port
      }"
    ];

    services.prometheus = {
      exporters.node.enable = config.prometheus.exporters.enable;
      exporters.node.enabledCollectors = [ ];
      exporters.node.disabledCollectors = [ "cpufreq" ];
      exporters.systemd.enable = config.prometheus.exporters.enable;
      exporters.process.enable = config.prometheus.exporters.enable;
      exporters.process.settings.process_names = [
        # Remove nix store path from process name
        {
          name = "{{.Matches.Wrapped}} {{ .Matches.Args }}";
          cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ];
        }
      ];
      extraFlags = lib.mkIf isServer [ "--web.enable-remote-write-receiver" ];
      scrapeConfigs = [{
        job_name = config.networking.hostName;
        static_configs = [{ targets = config.scrapeTargets; }];
      }];
      webExternalUrl =
        lib.mkIf isServer "https://${config.hostnames.prometheus}";
      # Web config file: https://prometheus.io/docs/prometheus/latest/configuration/https/
      webConfigFile = lib.mkIf isServer
        ((pkgs.formats.yaml { }).generate "webconfig.yml" {
          basic_auth_users = {
            # Generate password: htpasswd -nBC 10 "" | tr -d ':\n'
            # Encrypt and place in private/prometheus.age
            "prometheus" =
              "$2y$10$r7FWHLHTGPAY312PdhkPEuvb05aGn9Nk1IO7qtUUUjmaDl35l6sLa";
          };
        });
      remoteWrite = lib.mkIf (!isServer) [{
        name = config.networking.hostName;
        url = "https://${config.hostnames.prometheus}/api/v1/write";
        basic_auth = {
          # Uses password hashed with bcrypt above
          username = "prometheus";
          password_file = config.secrets.prometheus.dest;
        };
      }];
    };

    # Create credentials file for remote Prometheus push
    secrets.prometheus =
      lib.mkIf (config.services.prometheus.enable && !isServer) {
        source = ../../../private/prometheus.age;
        dest = "${config.secretsDirectory}/prometheus";
        owner = "prometheus";
        group = "prometheus";
        permissions = "0440";
      };
    systemd.services.prometheus-secret =
      lib.mkIf (config.services.prometheus.enable && !isServer) {
        requiredBy = [ "prometheus.service" ];
        before = [ "prometheus.service" ];
      };

    caddy.routes = lib.mkIf (config.services.prometheus.enable && isServer) [{
      match = [{ host = [ config.hostnames.prometheus ]; }];
      handle = [{
        handler = "reverse_proxy";
        upstreams =
          [{ dial = "localhost:${config.services.prometheus.port}"; }];
      }];
    }];

    # Configure Cloudflare DNS to point to this machine
    services.cloudflare-dyndns.domains =
      if (config.services.prometheus.enable && isServer) then
        [ config.hostnames.prometheus ]
      else
        [ ];

  };

}