.forgejo/workflows/eval.yaml

@@ -69,20 +69,139 @@ jobs:
           # Enter the shell
           nix-shell --run 'colmena build --on rescue01'
 
-  push_to_cache:
+  build_geo01:
+    runs-on: nix
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build geo01
+        run: |
+          # Enter the shell
+          nix-shell --run 'colmena build --on geo01'
+
+  build_geo02:
+    runs-on: nix
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build geo02
+        run: |
+          # Enter the shell
+          nix-shell --run 'colmena build --on geo02'
+
+  push_to_cache_compute01:
     runs-on: nix
     needs:
       - build_compute01
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Push to cache
+        run: nix-shell --run push-to-nix-cache
+        env:
+          STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/"
+          STORE_USER: "admin"
+          STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }}
+          NODES: '[ "compute01" ]'
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: outputs_compute01
+          path: uploaded.txt
+
+  push_to_cache_storage01:
+    runs-on: nix
+    needs:
       - build_storage01
-      - build_vault01
+    steps:
-      - build_web01
+      - uses: actions/checkout@v3
-      - build_web02
+
+      - name: Push to cache
+        run: nix-shell --run push-to-nix-cache
+        env:
+          STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/"
+          STORE_USER: "admin"
+          STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }}
+          NODES: '[ "storage01" ]'
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: outputs_storage01
+          path: uploaded.txt
+
+  push_to_cache_rescue01:
+    runs-on: nix
+    needs:
       - build_rescue01
     steps:
       - uses: actions/checkout@v3
 
       - name: Push to cache
-        run: nix-shell --run push-to-cache
+        run: nix-shell --run push-to-nix-cache
         env:
-          ATTIC_ENDPOINT: "https://cachix.dgnum.eu"
+          STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/"
-          ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
+          STORE_USER: "admin"
+          STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }}
+          NODES: '[ "rescue01" ]'
+
+  push_to_cache_geo01:
+    runs-on: nix
+    needs:
+      - build_geo01
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Push to cache
+        run: nix-shell --run push-to-nix-cache
+        env:
+          STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/"
+          STORE_USER: "admin"
+          STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }}
+          NODES: '[ "geo01" ]'
+
+  push_to_cache_geo02:
+    runs-on: nix
+    needs:
+      - build_geo02
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Push to cache
+        run: nix-shell --run push-to-nix-cache
+        env:
+          STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/"
+          STORE_USER: "admin"
+          STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }}
+          NODES: '[ "geo02" ]'
+
+  push_to_cache_web01:
+    runs-on: nix
+    needs:
+      - build_web01
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Push to cache
+        run: nix-shell --run push-to-nix-cache
+        env:
+          STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/"
+          STORE_USER: "admin"
+          STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }}
+          NODES: '[ "web01" ]'
+
+  push_to_cache_web02:
+    runs-on: nix
+    needs:
+      - build_web02
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Push to cache
+        run: nix-shell --run push-to-nix-cache
+        env:
+          STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/"
+          STORE_USER: "admin"
+          STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }}
+          NODES: '[ "web02" ]'

machines/storage01/_configuration.nix

@@ -11,6 +11,7 @@ lib.extra.mkConfig {
   enabledServices = [
     # List of services to enable
     "atticd"
+    "tvix-cache"
     "forgejo"
     "forgejo-runners"
     "garage"

machines/storage01/secrets/nginx-tvix-store-password

@@ -0,0 +1,28 @@
+age-encryption.org/v1
+-> ssh-ed25519 jIXfPA hiozo++fCkzjrvUQRLnAh4uwlmIXcTwkVbjkYbcH4mQ
+boST8EzrWdNAuyOylbBX//DnWtO7RL2W++Wnm40w2MA
+-> ssh-ed25519 QlRB9Q i0StXRfRRlTsN7MNZmlfBQdacHQlmTmriyiRcJu74g0
+dhkD9ZfW+mkkryHBu+2fHe76hXrWVGKl+orxkPJD6gU
+-> ssh-ed25519 r+nK/Q Ekn/Bz+c+G+KwgZEOCdk58lV9XN12d7/f+wi8ZEysgU
+QdvnL+HtpHnxUbKD06WZDAi55q3xOYn3OiHViNdFt+I
+-> ssh-rsa krWCLQ
+ijGL8v8Otp59VvF0tDIReazFzchihsutr+zbcQuB6m3JZ6SAWyoKwhFdwiaLOfUd
+DMAo2FOKfCbWS+M1VpdSJfu9LKroMCkeW+FOK81h6ywEYSAw/vt2FJP2TLiljZou
+d7hiqNv0u/yiIoQiTs9hwOAPtLofiWcX//18TNTCgqm9Ttn0mKlfBjTkUQJdkZVM
+j1rofzgHDdkyZDdr1op3sc4iURJ98dVN7ic035Fz+Ggs0yBh9T7qtVsUe7swuoH9
+b9yxOSHdV3b4BYg75UrfiRNTOeQq8pxsga1DIs2x7oHkeVb8Ypmr1tXuAtWi20eg
+1cYP5+BxY8ry6uaYNLYpKw
+-> ssh-ed25519 /vwQcQ ZuVSKV4sI53zDaTOHIkk6ntPy9IxSBNIN/JEDPfT71Y
+C5UgzlDJCcA8CP5D0kppqJKti76qe5IVFFnNirRtl/s
+-> ssh-ed25519 0R97PA bNQCB3PAp5Ka2drYm74R7nuGM7NFUsKluPo6EEEyiVA
+1/NFavNSG1pdMiWr2q2z9XwHs6iqhh5+3KIlr8ToPOo
+-> ssh-ed25519 JGx7Ng 6X2a/FNvglr8ZSWvgEb37B67JJpJV0x1+fdlo6K6pzo
+8AxYhMJ5+XGKNnpRBTSUM4GSbRj8s7amMQa8sp+tQWM
+-> ssh-ed25519 5SY7Kg xw7EQG3mz6gQZXSh2LpY5zFRyMZOqEypvnOorRLBBHQ
+WTcl4rLfg/siaGFmk/Odc6fsX+C6OPRWTHFQ0eENwgY
+-> ssh-ed25519 p/Mg4Q hSz69OeCJyLJIpnI1tJqGNRErbDF2v6OdxWxi/pfF3k
+nM6aJWcuzXEqRarkkAQx4636bALK3g0AwCsSfc8fXrk
+-> ssh-ed25519 rHotTw xyrUv1xRQGG+CyL7Ftdw50S8LtN3Bd07f+8JInmBdGg
+ehZkeby649QdiSyCDP4wTplLU7mtXac9QzILFIkIX/8
+--- xWjuc/9B2UAHi7vuOjdvwJ2K3MEeDeTon5XDU1zi6rw
+i«(rçfJ!–G
$<24>e)¤êý¡é•%)„‚9<>KÙ®UK¿Ëé]oǹË@Âv<C382>ŒÀ2I­pè\<12>ˆ^©9ä]¿ÂL,Ÿ•5æö/wvYŽÒ<C5BD>Í«‡³¬¼

machines/storage01/secrets/secrets.nix

@@ -13,10 +13,13 @@ lib.setDefault { inherit publicKeys; } [
   "influxdb2-initial_token_file"
   "influxdb2-telegraf_token_file"
   "netbird-auth_client_secret_file"
+  "nginx-tvix-store-password"
+  "nginx-tvix-store-password-ci"
   "peertube-secrets_file"
   "peertube-service_environment_file"
   "peertube-smtp_password_file"
-  "prometheus-web_config_file"
   "prometheus-garage_api"
   "prometheus-uptime-kuma-apikey"
+  "prometheus-web_config_file"
+  "tvix-store-infra-signing-key"
 ]

machines/storage01/secrets/tvix-store-infra-signing-key

@@ -0,0 +1,29 @@
+age-encryption.org/v1
+-> ssh-ed25519 jIXfPA /4nTbCIrufpN0Jho+8ZqTdZpc8mzSQrpG78flq+b9lM
+x6Pg9oMGzboBg4WSAHxPwtNKcJUIG007Wx1ZjlzneLc
+-> ssh-ed25519 QlRB9Q LsPsxbx6zvcLNf/EC3yFRP7Gr5tLYcg+8WGx6n0S724
+4cyAHEdVBR885G4nfJSvUPqKWr/0abAtDTHmwksADp8
+-> ssh-ed25519 r+nK/Q 9MisKxWalh0oubQFjwm2SDggxrj/fhdXGCYuYaP99jA
+18o9juckqPtR4gh2MTXdmonxV9oZymyhCUqW3sOVltQ
+-> ssh-rsa krWCLQ
+j6AIypswOisUPlL538E3dpIWsHU/7H1c3+bEXXDFarP3Y5tjWltMRgKoPZUFlcRk
+2yoVpOjDVkDvMTTu62Yn+Le6oYqoYQYzZ4e5incAR/v7sI76yPo1w+JN3BWBKPab
+DN6h7Bdr8uzMISvxrRpCNDaU9n9GwA6ylJWvtFKjQZ6IDORVsa1tP44cndm6zAt6
+Oq11bUDFSJLHiDtxjp0vJFa/4mq5Ay0G10xM/EI8Wf+Tiam/r3ytoBGnNYj1ENp8
+AQkSxVF4cCORjQAokg+eUYCOzErJqpOx0ACx1SvuRvG4qcQ55ChYxs9zjnlCII2x
+7JeUM/gjy0FnalxWWDX+cQ
+-> ssh-ed25519 /vwQcQ bdzz3o+erI4c7ReafjhMYBgpebcJVcdB5vWK7cQ05Cs
+3rVELKWfeiBksMzmm9XLmEgzdEASxSKcYJOpDQd7A+w
+-> ssh-ed25519 0R97PA 4k2mZBQJTYhbjdzpxDuNw405iNxd96hVSMwzas/D3nU
+neRy8ca2SguOJJQxalbPaq5SUH4taH+XxzkU/o/GVig
+-> ssh-ed25519 JGx7Ng BlMr9FS9vuC1wnvDBAqEMJWzyuqoMqoU7YiFC9633xo
+Xhvn+luDLE7AFbvgJs6V9cyRh8aJ2JrZfpVvXJhclu4
+-> ssh-ed25519 5SY7Kg NkkDnN0z+2EzqpEdypnM7AROjjGVzoEvHfzaVbsyDiE
+qbFUDBx4ghp9TG9YfjGjDXt35go0pMq0HH9GE+WT4v8
+-> ssh-ed25519 p/Mg4Q rC/DrdXDUDWhbM7LMfQR203JClF/12o4rxJeGs+4rXY
+Aj3P3skTbMvt2qN/FPSq97D1QwtHlKvFd4CsoujV2JI
+-> ssh-ed25519 rHotTw 5IBV+q7+F7vNs5Tsx0S+ZEstiqoAaH1x78i/vAwrwDw
+f729cEfMo/ozygHiRcNXmn8G+M+B68cM48ji7N6VgmY
+--- TWScQDjdR4g/2v5oirYJgQw4zhhuMnmfvXtrigwmZC4
+é°1ØLÅÄ‘ßán`Îq^ˆîÚ<C3AE>ï³Q²,ðT«Ó)Lñaü„226M•‘¿Éú½Ü~››4<E280BA>(~’e±.®Y"´M·×!Žp!ÊU<ÖÜŒ–<C592>Â;mn§`,öP–6*&}HPM‡I¶º
òïH
+Ûôï×Ãmõ<6D>‡ m£<6D>dGÎ	߆ß÷T¥?G<>É»/

machines/storage01/tvix-cache/default.nix

@@ -0,0 +1,148 @@
+{ pkgs, config, ... }:
+let
+  settingsFormat = pkgs.formats.toml { };
+
+  dataDir = "/data/slow/tvix-store";
+
+  store-config = {
+    composition = {
+      blobservices.default = {
+        type = "objectstore";
+        object_store_url = "file://${dataDir}/blob.objectstore";
+        object_store_options = { };
+      };
+      directoryservices = {
+        sled = {
+          type = "sled";
+          is_temporary = false;
+          path = "${dataDir}/directory.sled";
+        };
+        object = {
+          type = "objectstore";
+          object_store_url = "file://${dataDir}/directory.objectstore";
+          object_store_options = { };
+        };
+      };
+      pathinfoservices = {
+        infra = {
+          type = "sled";
+          is_temporary = false;
+          path = "${dataDir}/pathinfo.sled";
+        };
+        infra-signing = {
+          type = "keyfile-signing";
+          inner = "infra";
+          keyfile = config.age.secrets."tvix-store-infra-signing-key".path;
+        };
+      };
+    };
+
+    endpoints = {
+      "127.0.0.1:8056" = {
+        endpoint_type = "Http";
+        blob_service = "default";
+        directory_service = "object";
+        path_info_service = "infra";
+      };
+      "127.0.0.1:8058" = {
+        endpoint_type = "Http";
+        blob_service = "default";
+        directory_service = "object";
+        path_info_service = "infra-signing";
+      };
+      # Add grpc for management and because it is nice
+      "127.0.0.1:8057" = {
+        endpoint_type = "Grpc";
+        blob_service = "default";
+        directory_service = "object";
+        path_info_service = "infra";
+      };
+    };
+  };
+  systemdHardening = {
+    PrivateDevices = true;
+    PrivateTmp = true;
+    ProtectControlGroups = true;
+    ProtectKernelTunables = true;
+    RestrictSUIDSGID = true;
+
+    ProtectSystem = "strict";
+    ProtectKernelLogs = true;
+    ProtectProc = "invisible";
+    PrivateUsers = true;
+    ProtectHome = true;
+    UMask = "0077";
+    RuntimeDirectoryMode = "0750";
+    StateDirectoryMode = "0750";
+  };
+  toml = {
+    composition = settingsFormat.generate "composition.toml" store-config.composition;
+    endpoints = settingsFormat.generate "endpoints.toml" store-config.endpoints;
+  };
+  package = pkgs.callPackage ./package { };
+in
+{
+
+  age-secrets.autoMatch = [
+    "tvix-store"
+    "nginx"
+  ];
+
+  services.nginx.virtualHosts."tvix-store.dgnum.eu" = {
+    enableACME = true;
+    forceSSL = true;
+    locations = {
+      "/infra/" = {
+        proxyPass = "http://127.0.0.1:8056/";
+        extraConfig = ''
+          client_max_body_size 50G;
+          limit_except GET {
+            auth_basic "Password required";
+            auth_basic_user_file ${config.age.secrets."nginx-tvix-store-password".path};
+          }
+        '';
+      };
+      "/infra-signing/" = {
+        proxyPass = "http://127.0.0.1:8058/";
+        extraConfig = ''
+          client_max_body_size 50G;
+          auth_basic "Password required";
+          auth_basic_user_file ${config.age.secrets."nginx-tvix-store-password-ci".path};
+        '';
+      };
+      "/.well-known/nix-signing-keys/" = {
+        alias = "${./pubkeys}/";
+        extraConfig = "autoindex on;";
+      };
+    };
+  };
+  # TODO add tvix-store cli here
+  # environment.systemPackages = [ ];
+  users.users.tvix-store = {
+    isSystemUser = true;
+    group = "tvix-store";
+  };
+  users.groups.tvix-store = { };
+
+  systemd.tmpfiles.rules = [ "d ${dataDir} 770 tvix-castore tvix-castore -" ];
+
+  systemd.services."tvix-store" = {
+    wantedBy = [ "multi-user.target" ];
+    environment = {
+      RUST_LOG = "debug";
+    };
+    serviceConfig = {
+      UMask = "007";
+      ExecStart = "${package}/bin/multitier-tvix-cache --endpoints-config ${toml.endpoints} --store-composition ${toml.composition}";
+      StateDirectory = "tvix-store";
+      RuntimeDirectory = "tvix-store";
+      User = "tvix-store";
+      Group = "tvix-store";
+      ReadWritePaths = [ dataDir ];
+    } // systemdHardening;
+  };
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+}

machines/storage01/tvix-cache/package/default.nix

@@ -0,0 +1,45 @@
+{
+  fetchgit,
+  rustPlatform,
+  protobuf,
+  runCommand,
+}:
+let
+  tvix-hash = "sha256-KNl+Lv0aMqSFVFt6p/GdmNDddzccW4wKfZB7W6Gv5F0=";
+  tvix-src = fetchgit {
+    name = "tvix";
+    url = "https://git.dgnum.eu/mdebray/tvl-depot";
+    rev = "920b7118d5b0917e426367107f7b7b66089a8d7b";
+    hash = tvix-hash;
+  };
+  protos = runCommand "tvix-protos" { } ''
+    mkdir $out
+    cd ${tvix-src}/tvix #remove tvix maybe
+    find . -name '*.proto' -exec install -D {} $out/{} \;
+  '';
+in
+
+rustPlatform.buildRustPackage rec {
+  pname = "multitenant-binary-cache";
+  version = "0.1.0";
+
+  src = fetchgit {
+    url = "https://git.lix.systems/sinavir/multitenant-tvix-binary-cache.git";
+    rev = "0d7d4cf66242facecba485b1085e285e8d46c038";
+    hash = "sha256-IU3OS3ePJeBNiY8HbhoYW5b03Nq8BJ4AWe+bGv4dAuw=";
+  };
+
+  PROTO_ROOT = protos;
+
+  nativeBuildInputs = [ protobuf ];
+
+  cargoLock = {
+    lockFile = ./Cargo.lock;
+    outputHashes = {
+      "nar-bridge-0.1.0" = tvix-hash;
+    };
+  };
+  cargoHash = "";
+
+  meta = { };
+}

machines/storage01/tvix-cache/pubkeys/infra

@@ -0,0 +1 @@
+infra.tvix-store.dgnum.eu-1:8CAY64o3rKjyw2uA5mzr/aTzstnc+Uj4g8OC6ClG1m8=

meta/dns.nix

@@ -67,6 +67,7 @@ let
 
         storage01.dual = [
           "cachix" # Attic
+          "tvix-store" # tvix store
           "git" # Forgejo
           "influx" # InfluxDB
           "netbird" # Netbird

scripts/default.nix

@@ -34,6 +34,7 @@ let
     "launch-vm"
     "list-nodes"
     "push-to-cache"
+    "push-to-nix-cache"
     "cache"
   ];
 in

scripts/push-to-cache.sh

@@ -8,6 +8,6 @@ ENDPOINT=${ATTIC_ENDPOINT:-https://cachix.dgnum.eu}
 
 @colmena@/bin/colmena eval -E '{ nodes, lib, ... }: lib.mapAttrsToList (_: v: v.config.system.build.toplevel.drvPath) nodes' |\
 @jq@/bin/jq -r '.[]' |\
-xargs nix-store -q -R --include-outputs |\
+xargs -n 10 nix-store -q -R --include-outputs |\
 sed '/\.drv$/d' |\
 xargs @attic@/bin/attic push dgnum:infra

scripts/push-to-nix-cache.sh

@@ -0,0 +1,20 @@
+set -e
+set -u
+set -o pipefail
+
+ENDPOINT=${STORE_ENDPOINT:-https://tvix-cache.dgnum.eu/infra-singing/}
+
+cat > .netrc << EOF
+default
+login $STORE_USER
+password $STORE_PASSWORD
+EOF
+
+@colmena@/bin/colmena eval -E "{ nodes, lib, ... }: builtins.map (v: nodes.\${v}.config.system.build.toplevel.drvPath) ${NODES:-(builtins.attrNames nodes)}" |\
+@jq@/bin/jq -r '.[]' |\
+xargs nix-store -q -R --include-outputs |\
+sed '/\.drv$/d' |\
+tee uploaded.txt |\
+xargs nix copy --to "$ENDPOINT?compression=none" --extra-experimental-features nix-command --netrc-file ./.netrc
+
+rm .netrc