e4adca0880
This adds a `parse-bucket-logs.{service,timer}`, running once every
night at 3AM UTC, figuring out the last time it was run and parsing
bucket logs for all previous days.
It invokes the `archeology-parse-bucket-logs` script to produce
a .parquet file with the bucket logs in `s3://nix-cache-log/log/` for
that day (inside a temporary directory), then on success uploads the
produced parquet file to
`s3://nix-archeologist/nix-cache-bucket-logs/yyyy-mm-dd.parquet`.
Change-Id: Ia75ca8c43f8074fbaa34537ffdba68350c504e52
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10011
Reviewed-by: edef <edef@edef.eu>
Tested-by: BuildkiteCI
35 lines
901 B
Nix
35 lines
901 B
Nix
{ depot, pkgs, modulesPath, ... }:
|
|
|
|
{
|
|
imports = [
|
|
"${modulesPath}/virtualisation/amazon-image.nix"
|
|
../profiles/archeology.nix
|
|
];
|
|
|
|
systemd.timers.parse-bucket-logs = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
timerConfig.OnCalendar = "*-*-* 03:00:00 UTC";
|
|
};
|
|
|
|
systemd.services.parse-bucket-logs = {
|
|
path = [ depot.users.flokli.archeology.parse-bucket-logs ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecStart = (pkgs.writers.writePython3 "parse-bucket-logs-continuously"
|
|
{
|
|
libraries = [ pkgs.python3Packages.boto3 ];
|
|
} ./parse-bucket-logs-continuously.py);
|
|
DynamicUser = "yes";
|
|
StateDirectory = "parse-bucket-logs";
|
|
};
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
depot.users.flokli.archeology.parse-bucket-logs
|
|
];
|
|
|
|
networking.hostName = "archeology-ec2";
|
|
|
|
system.stateVersion = "23.05"; # Did you read the comment?
|
|
}
|
|
|