noah/dotfiles
1
0
Fork 0
mirror of https://github.com/nmasur/dotfiles synced 2024-11-22 06:25:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

provision grafana alerts and smtp

Browse Source
This commit is contained in:
Noah Masur 2023-08-05 14:21:49 +00:00
parent de1337bf58
commit b7d1921c4d
2 changed files with 173 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

166
modules/nixos/services/grafana.nix
View File

@ -7,12 +7,34 @@ in {
config = lib.mkIf config.services.grafana.enable {
secrets.mailpass-grafana = {
source = ../../../private/mailpass-grafana.age;
dest = "${config.secretsDirectory}/mailpass-grafana";
owner = "grafana";
group = "grafana";
permissions = "0440";
};
systemd.services.mailpass-grafana-secret = {
requiredBy = [ "grafana.service" ];
before = [ "grafana.service" ];
};
services.grafana = {
settings.server = {
domain = config.hostnames.metrics;
http_addr = "127.0.0.1";
http_port = 3000;
protocol = "http";
settings = {
server = {
domain = config.hostnames.metrics;
http_addr = "127.0.0.1";
http_port = 3000;
protocol = "http";
};
smtp = rec {
enabled = true;
host = "${config.mail.smtpHost}:465";
user = "grafana@${config.mail.server}";
password = "$__file{${config.secrets.mailpass-grafana.dest}}";
from_name = "Grafana";
from_address = user;
};
};
provision = {
enable = true;
@ -2156,6 +2178,140 @@ in {
}))
}/dashboards";
}];
alerting = {
contactPoints.settings.contactPoints = [{
name = "grafana-default-email";
receivers = [{
uid = "basic-email";
type = "email";
settings.addresses = "grafana@${config.mail.server}";
}];
}];
muteTimings = { };
policies.settings = { resetPolicies = [ 1 ]; };
rules.settings.groups = [{
name = "Default";
interval = "1m";
folder = "Alerts";
rules = [{
uid = "cloudflare-tunnel";
title = "Cloudflare Tunnel";
condition = "C";
data = [
# Query to retrieve the status data
{
refId = "A";
relativeTimeRange = {
from = 600;
to = 0;
};
datasourceUid = promUid;
model = {
editorMode = "code";
expr = ''
systemd_unit_state{name=~"cloudflared-tunnel-.*", state="active"}'';
hide = false;
instant = true;
intervalMs = 1000;
maxDataPoints = 43200;
range = false;
refId = "A";
};
}
# Reduce to the max level, to ensure no false alarms
{
refId = "B";
relativeTimeRange = {
from = 600;
to = 0;
};
datasourceUid = "__expr__";
model = {
conditions = [{
evaluator = {
params = [ ];
type = "gt";
};
operator = { type = "and"; };
query = { params = [ "B" ]; };
reducer = {
params = [ ];
type = "last";
};
type = "query";
}];
datasource = {
type = "__expr__";
uid = "__expr__";
};
expression = "A";
hide = false;
intervalMs = 1000;
maxDataPoints = 43200;
reducer = "max";
refId = "B";
type = "reduce";
};
}
# Threshold to trigger alarm if below 100% uptime
{
refId = "C";
relativeTimeRange = {
from = 600;
to = 0;
};
datasourceUid = "__expr__";
model = {
conditions = [{
evaluator = {
params = [ 1 ];
type = "lt";
};
operator = { type = "and"; };
query = { params = [ "C" ]; };
reducer = {
params = [ ];
type = "last";
};
type = "query";
}];
datasource = {
type = "__expr__";
uid = "__expr__";
};
expression = "B";
hide = false;
intervalMs = 1000;
maxDataPoints = 43200;
refId = "C";
type = "threshold";
};
}
];
noDataState = "Alerting";
execErrState = "Error";
for = "5m";
annotations = {
description = "Cloudflare Tunnel for {{ $job }}.";
summary = "Cloudflare Tunnel is down.";
};
isPaused = false;
}];
}];
templates = { };
};
# notifiers = [];
};
};

12
private/mailpass-grafana.age Normal file
View File

@ -0,0 +1,12 @@
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IE1nSGFPdyBrWGI1
bGNETlp2VjJNVGJaQjNFVHc2S2xmd1dYOFhLaXNKSzVucXNJZzNnClhpYThLeHdW
NUIxTWhTVWdhTTBkU2VSMDk5ZEtNQnJESFpTbEdGMEkvZVkKLT4gc3NoLWVkMjU1
MTkgWXlTVU1RIDNHazBJN0lMSmFqVnBabjFlQlZQb3EydUlZZkhsUkd1QXRQV24r
Mlk4MjgKbWJnckplbm1yZHdOYklRNXg4VURLTUhyZS91SVdpVmM1eGdLNEhqaUhx
OAotPiBzc2gtZWQyNTUxOSBuanZYNUEgcUNzKzMyT0I0NDVnUmlidXNkMG1WYi93
Q2JHbE1XRjc2cWljdE9kOGRudwpPTUljRFJMVU0xQmlYdW5WZXM1ZTVjZVNJcXpL
QkVTYU1WM1lTMVRDSk13Ci0tLSBYZzNnQTMzTHk5R2hIZ0I1Y1J5R3JzM0Q3Sklz
a2JoYUxuNzZhdXJ5LzZNCt9/PL3SEFcF46B3zk5VqsTvlGbIxjbD9fZj66vAt6G9
o/xsHsWign4o4rBMyL52hNE8uZs=
-----END AGE ENCRYPTED FILE-----