mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(aspen/system): Move metrics to ogopogo, refresh

Browse source 
Change-Id: I93ddc961b473e15febe22a16879875dbd926236a
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12501
Autosubmit: aspen <root@gws.fyi>
Tested-by: BuildkiteCI
Reviewed-by: aspen <root@gws.fyi>
This commit is contained in:
Aspen Smith 2024-09-21 13:30:44 -04:00 committed by clbot
parent bf4f1a43e5
commit a15760671d
13 changed files with 240 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
ops/machines/all-systems.nix
View file

@ -4,7 +4,6 @@
sanduny
whitby
nixery-01
volgasprint-cache
]) ++
(with depot.users.tazjin.nixos; [

BIN
users/aspen/secrets/bbbg.age
View file

Binary file not shown.

BIN
users/aspen/secrets/buildkite-ssh-key.age
View file

Binary file not shown.

BIN
users/aspen/secrets/buildkite-token.age
View file

Binary file not shown.

BIN
users/aspen/secrets/cloudflare.age
View file

Binary file not shown.

BIN
users/aspen/secrets/ddclient-password.age
View file

Binary file not shown.

2
users/aspen/secrets/secrets.nix
View file

@ -7,7 +7,7 @@ in
{
"bbbg.age".publicKeys = [ grfn mugwump bbbg ];
"cloudflare.age".publicKeys = [ grfn mugwump ];
"cloudflare.age".publicKeys = [ grfn mugwump ogopogo ];
"ddclient-password.age".publicKeys = [ grfn ogopogo ];
"buildkite-ssh-key.age".publicKeys = [ grfn mugwump ogopogo ];
"buildkite-token.age".publicKeys = [ grfn mugwump ogopogo ];

16
users/aspen/secrets/windtunnel-bot-github-token.age
View file

@ -1,9 +1,9 @@
age-encryption.org/v1
-> ssh-ed25519 CpJBgQ qVlQpHyewtBSfFIdU8GihXC7JhGbcvQ61ZsJC20wSH4
mZXwiTICzrG+3aCL67cO6cTWMgHkxhDyBi7tZ8l+QMA
-> ssh-ed25519 LfBFbQ 78NQxflRkRMW5vSP1BEvASSQU2pZAfMwd7T2+6W7NQs
u0x986pFtnD9ZqfL3KnRrdYS5z9LRUPJhcmc8FQOuGo
-> ssh-ed25519 GeE7sQ aqFQGCywSimHNbN5si0PzmESUXwROjrpTe/5UdTyYw4
X2thEJIyOnNUsA746VwqZhH+44XBfCTvh7VOEg/zew0
--- ndSgjJv5Tel6ovKl+SBdDHZHlszgsEhOY1HHpNDvf1s
ÒüI¼Êʵu*1ðÄt©(úùºîƒ/œXÀÜË•3È’<C388>ï<12>†VGúÁT|Î@<40>·ÌKó¾<}§)se¹9`í¶*z
-> ssh-ed25519 CpJBgQ PiY6IidA+GRbpjL91BVe9UdejWvi02SRcijiMOjXcm4
XegOhgjdEdzXtz31PsGVyOZ10gH6P82Q1/txZcSxjIY
-> ssh-ed25519 LfBFbQ uqRF0nKMk1GrK+6pEBdmyHKu2ewDFlWwlKC+myey4gc
dgnX4eprSolXxCDNoVmGzGK9xLEmtmeg/cJihD4/8sU
-> ssh-ed25519 GeE7sQ ikAIyFR/qH1a+aa5mumiiDwa5o5aLsQeJKwQwMzgs1M
8htzhM5t2VnjRBrC+VrL23f9chlQjVGzjxMaFB7Arrs
--- Qm16HTo5wGUBKS0ly3OZDWp2etLyDS/zlxOHxPjS8PI
ƒ¤7¦–NY6Âk|‡p2·'–˜&Õ=‚m£Çq`5ÿT¸ ëNÕ9ÖNϵÖÏ)RVºU-é•) ¡¼M<C2BC>‰¢(%pœ

1
users/aspen/system/system/machines/lusca.nix
View file

@ -10,6 +10,7 @@
../modules/sound.nix
../modules/tvl.nix
../modules/development.nix
../modules/prometheus-exporter.nix
];
networking.hostName = "lusca";

140
users/aspen/system/system/machines/mugwump.nix
View file

@ -117,149 +117,9 @@ with lib;
};
};
services.grafana = {
enable = true;
dataDir = "/var/lib/grafana";
settings = {
server = {
http_port = 3000;
root_url = "https://metrics.gws.fyi";
domain = "metrics.gws.fyi";
};
analytics.reporting_enabled = false;
};
provision = {
enable = true;
datasources.settings.datasources = [{
name = "Prometheus";
type = "prometheus";
url = "http://localhost:9090";
}];
};
};
security.acme.defaults.email = "root@gws.fyi";
security.acme.acceptTerms = true;
services.nginx = {
enable = true;
statusPage = true;
recommendedGzipSettings = true;
recommendedOptimisation = true;
recommendedTlsSettings = true;
recommendedProxySettings = true;
virtualHosts = {
"metrics.gws.fyi" = {
enableACME = true;
forceSSL = true;
locations."/" = {
proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
};
};
};
};
security.acme.certs."metrics.gws.fyi" = {
dnsProvider = "cloudflare";
credentialsFile = config.age.secretsDir + "/cloudflare";
webroot = mkForce null;
};
services.prometheus = {
enable = true;
exporters = {
node = {
enable = true;
openFirewall = false;
enabledCollectors = [
"processes"
"systemd"
"tcpstat"
"wifi"
];
};
nginx = {
enable = true;
openFirewall = true;
sslVerify = false;
constLabels = [ "host=mugwump" ];
};
blackbox = {
enable = true;
openFirewall = true;
configFile = pkgs.writeText "blackbox-exporter.yaml" (builtins.toJSON {
modules = {
https_2xx = {
prober = "http";
http = {
method = "GET";
fail_if_ssl = false;
fail_if_not_ssl = true;
preferred_ip_protocol = "ip4";
};
};
};
});
};
};
scrapeConfigs = [
{
job_name = "node";
scrape_interval = "5s";
static_configs = [{
targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
}];
}
{
job_name = "nginx";
scrape_interval = "5s";
static_configs = [{
targets = [ "localhost:${toString config.services.prometheus.exporters.nginx.port}" ];
}];
}
{
job_name = "xanthous_server";
scrape_interval = "1s";
static_configs = [{
targets = [ "localhost:${toString config.services.xanthous-server.metricsPort}" ];
}];
}
{
job_name = "blackbox";
metrics_path = "/probe";
params.module = [ "https_2xx" ];
scrape_interval = "5s";
static_configs = [{
targets = [
"https://gws.fyi"
"https://windtunnel.ci"
"https://app.windtunnel.ci"
"https://metrics.gws.fyi"
];
}];
relabel_configs = [{
source_labels = [ "__address__" ];
target_label = "__param_target";
}
{
source_labels = [ "__param_target" ];
target_label = "instance";
}
{
target_label = "__address__";
replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
}];
}
];
};
services.xanthous-server.enable = true;
virtualisation.docker = {

3
users/aspen/system/system/machines/ogopogo.nix
View file

@ -11,6 +11,8 @@
../modules/tvl.nix
../modules/development.nix
../modules/wireshark.nix
../modules/metrics.nix
../modules/prometheus-exporter.nix
];
networking.hostName = "ogopogo";
@ -92,7 +94,6 @@
dataDir = "/data/postgresql";
package = pkgs.postgresql_15;
settings = {
port = 5431;
wal_level = "logical";
};
};

197
users/aspen/system/system/modules/metrics.nix Normal file
View file

@ -0,0 +1,197 @@
{ depot, config, lib, pkgs, ... }:
with lib;
let
nodesToScrape = [
"ogopogo"
# "dobharchu"
"mugwump"
# "yeren"
"lusca"
];
nodesRunningNginx = [
"ogopogo"
"mugwump"
];
nodesRunningPostgres = [
"ogopogo"
];
blackboxTargets = [
"https://gws.fyi"
"https://windtunnel.ci"
"https://app.windtunnel.ci"
"https://metrics.gws.fyi"
];
in
{
imports = [
(depot.third_party.agenix.src + "/modules/age.nix")
];
config = {
services.postgresql = {
ensureUsers = [{
name = config.services.grafana.settings.database.user;
ensureDBOwnership = true;
}];
ensureDatabases = [
config.services.grafana.settings.database.name
];
};
services.grafana = {
enable = true;
dataDir = "/var/lib/grafana";
settings = {
server = {
http_port = 3000;
root_url = "https://metrics.gws.fyi";
domain = "metrics.gws.fyi";
};
analytics.reporting_enabled = false;
database = {
type = "postgres";
user = "grafana";
name = "grafana";
host = "/run/postgresql";
};
};
provision = {
enable = true;
datasources.settings.datasources = [{
name = "Prometheus";
type = "prometheus";
url = "http://localhost:9090";
}];
};
};
security.acme.defaults.email = "root@gws.fyi";
security.acme.acceptTerms = true;
services.nginx = {
enable = true;
statusPage = true;
recommendedGzipSettings = true;
recommendedOptimisation = true;
recommendedTlsSettings = true;
recommendedProxySettings = true;
virtualHosts = {
"metrics.gws.fyi" = {
enableACME = true;
forceSSL = true;
locations."/" = {
proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
};
};
};
};
age.secrets = {
cloudflare.file = depot.users.aspen.secrets."cloudflare.age";
};
security.acme.certs."metrics.gws.fyi" = {
dnsProvider = "cloudflare";
credentialsFile = config.age.secretsDir + "/cloudflare";
webroot = mkForce null;
};
services.prometheus = {
enable = true;
retentionTime = "30d";
exporters = {
blackbox = {
enable = true;
openFirewall = true;
configFile = pkgs.writeText "blackbox-exporter.yaml" (builtins.toJSON {
modules = {
https_2xx = {
prober = "http";
http = {
method = "GET";
fail_if_ssl = false;
fail_if_not_ssl = true;
preferred_ip_protocol = "ip4";
};
};
};
});
};
};
scrapeConfigs = [
{
job_name = "node";
scrape_interval = "5s";
static_configs =
map
(node: {
targets = [ "${node}:${toString config.services.prometheus.exporters.node.port}" ];
labels.node = node;
})
nodesToScrape;
}
{
job_name = "nginx";
scrape_interval = "5s";
static_configs =
map
(node: {
targets = [ "${node}:${toString config.services.prometheus.exporters.nginx.port}" ];
labels.node = node;
})
nodesRunningNginx;
}
{
job_name = "postgres";
scrape_interval = "5s";
static_configs =
map
(node: {
targets = [ "${node}:${toString config.services.prometheus.exporters.postgres.port}" ];
labels.node = node;
})
nodesRunningPostgres;
}
{
job_name = "blackbox";
metrics_path = "/probe";
params.module = [ "https_2xx" ];
scrape_interval = "5s";
static_configs = [{
targets = [
"https://gws.fyi"
"https://windtunnel.ci"
"https://app.windtunnel.ci"
"https://metrics.gws.fyi"
];
}];
relabel_configs = [
{
source_labels = [ "__address__" ];
target_label = "__param_target";
}
{
source_labels = [ "__param_target" ];
target_label = "instance";
}
{
target_label = "__address__";
replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
}
];
}
];
};
};
}

31
users/aspen/system/system/modules/prometheus-exporter.nix Normal file
View file

@ -0,0 +1,31 @@
{ config, lib, pkgs, ... }:
with lib;
{
services.prometheus.exporters = {
node = {
enable = true;
openFirewall = false;
enabledCollectors = [
"processes"
"systemd"
"tcpstat"
"wifi"
];
};
nginx = mkIf config.services.nginx.enable {
enable = true;
openFirewall = true;
sslVerify = false;
constLabels = [ "host=${config.networking.hostName}" ];
};
postgres = mkIf config.services.postgresql.enable {
enable = true;
runAsLocalSuperUser = true;
};
};
}