infrastructure/machines/storage01/tvix-cache/default.nix

{ pkgs, config, ... }:
let

  # How to add a cache:
  #  - Add the relevant services (likely only a pathinfoservice) to the
  #    composition config (store-config.composition).
  #  - Add an endpoint (store-config.endpoints).
  #  - Append a proxy configuration to nginx in order to make the store
  #    accessible.
  #  - Update cache-info.nix so users can add the cache to their configuration
  store-config = {
    composition = {
      blobservices.default = {
        type = "objectstore";
        object_store_url = "file://${dataDir}/blob.objectstore";
        object_store_options = { };
      };
      directoryservices = {
        redb = {
          type = "redb";
          is_temporary = false;
          path = "${dataDir}/directory.redb";
        };
      };
      pathinfoservices = {
        infra = {
          type = "redb";
          is_temporary = false;
          path = "${dataDir}/pathinfo.redb";
        };
        infra-signing = {
          type = "keyfile-signing";
          inner = "infra";
          keyfile = config.age.secrets."tvix-store-infra-signing-key".path;
        };
      };
    };

    endpoints = {
      "127.0.0.1:8056" = {
        endpoint_type = "Http";
        blob_service = "default";
        directory_service = "redb";
        path_info_service = "infra";
      };
      "127.0.0.1:8058" = {
        endpoint_type = "Http";
        blob_service = "default";
        directory_service = "redb";
        path_info_service = "infra-signing";
      };
      # Add grpc for management and because it is nice
      "127.0.0.1:8057" = {
        endpoint_type = "Grpc";
        blob_service = "default";
        directory_service = "redb";
        path_info_service = "infra";
      };
    };
  };

  settingsFormat = pkgs.formats.toml { };

  webHost = "tvix-store.dgnum.eu";

  dataDir = "/data/slow/tvix-store";

  systemdHardening = {
    PrivateDevices = true;
    PrivateTmp = true;
    ProtectControlGroups = true;
    ProtectKernelTunables = true;
    RestrictSUIDSGID = true;

    ProtectSystem = "strict";
    ProtectKernelLogs = true;
    ProtectProc = "invisible";
    PrivateUsers = true;
    ProtectHome = true;
    UMask = "0077";
    RuntimeDirectoryMode = "0750";
    StateDirectoryMode = "0750";
  };

  toml = {
    composition = settingsFormat.generate "composition.toml" store-config.composition;
    endpoints = settingsFormat.generate "endpoints.toml" store-config.endpoints;
  };

  package = pkgs.callPackage ./package { };
in
{

  age-secrets.autoMatch = [
    "tvix-store"
    "nginx"
  ];

  services.nginx.virtualHosts.${webHost} = {
    enableACME = true;
    forceSSL = true;
    locations = {
      "/infra/" = {
        proxyPass = "http://127.0.0.1:8056/";
        extraConfig = ''
          client_max_body_size 50G;
          limit_except GET {
            auth_basic "Password required";
            auth_basic_user_file ${config.age.secrets."nginx-tvix-store-password".path};
          }
        '';
      };
      "/infra-signing/" = {
        proxyPass = "http://127.0.0.1:8058/";
        extraConfig = ''
          client_max_body_size 50G;
          auth_basic "Password required";
          auth_basic_user_file ${config.age.secrets."nginx-tvix-store-password-ci".path};
        '';
      };
    };
  };

  # TODO add tvix-store cli here
  # environment.systemPackages = [ ];

  users.users.tvix-store = {
    isSystemUser = true;
    group = "tvix-store";
  };
  users.groups.tvix-store = { };

  systemd.tmpfiles.rules = [ "d ${dataDir} 770 tvix-castore tvix-castore -" ];

  systemd.services."tvix-store" = {
    wantedBy = [ "multi-user.target" ];
    environment = {
      RUST_LOG = "info";
    };
    serviceConfig = {
      UMask = "007";
      LimitNOFILE = 1048576;
      ExecStart = "${package}/bin/multitier-tvix-cache --endpoints-config ${toml.endpoints} --store-composition ${toml.composition}";
      StateDirectory = "tvix-store";
      RuntimeDirectory = "tvix-store";
      User = "tvix-store";
      Group = "tvix-store";
      ReadWritePaths = [ dataDir ];
    } // systemdHardening;
  };
  networking.firewall.allowedTCPPorts = [
    80
    443
  ];
}