feat(infra): add S3 declarative buckets

A very simple basic support for it, which requires a S3 admin token.

Signed-off-by: Ryan Lahfa <ryan@dgnum.eu>

.credentials/admin-environment.age

@@ -0,0 +1,27 @@
+age-encryption.org/v1
+-> ssh-ed25519 jIXfPA sIWcGSvcykz9kOFJYILgAm5Xq2PBInCzJP4T2pJKcCE
+/lLwSyz7O2GcrL/m8tLswJxqUHWjMmixnZgaMIbcJlU
+-> ssh-ed25519 QlRB9Q tKNkVMqaxZjjxTTYUiAUMYb+br7bNNpeBU0sacEFchw
+NHBW4eOGqqstzN5flNPPYCSq1gZSXgqWceXuxFvKMFE
+-> ssh-ed25519 r+nK/Q ULxI2nmWCE3/k8zlD40HrfIRXHGDSIJn0WWTBSUfqAU
+27Yp/f6Kq/xxhN2TBfsm+IcowXPA7258mqwOHCWwtNo
+-> ssh-rsa krWCLQ
+cLJmGyyl8JnQhnhqUYLy5nGy83aGB1CeuOSOzuxpUHutnwprbTCQbQDDmMHepzkE
+gO+vqX95E3Y1XH916kV6+0IfgrbUdN3HdQylquHAwrW47en/Nmcnzqmn2eLZv6AJ
+o+HvxlVIj6TnQehuidVPZN1uppgKnmwvlcle9MmaRwCGlC/Ysvpa1yn88uC26wv8
+7b8ONBen25iJaQ79w7f5J9bkoCHQ3hhWQfv1ZrSOfyqwj4L4AimaIRnFRmPLhxma
+wjOBOPA4+ibBhxy0eexKRM5pWuQ6+iY/j6bJyQoEr+TqZADORRAmP7FhC3Uync7f
+TWPZz5kZYsuk4TI4Fv81aw
+-> ssh-ed25519 /vwQcQ 0XpYJjvhUbOdLJ0aQVckn3nx6FTKp+Gf4i3670XnyWQ
+PByb00P+4rE+hwehIgAj35YPdaTebvoljfhYHZQEMcY
+-> ssh-ed25519 0R97PA 9xIPfaZO82LKBHivlhuwXwcV/ZkJzKyQgSlRrlxx/WM
+abN/QW+CJsLvmTPNGoVygcKTHUzAmzedYkgWMl6IXWA
+-> ssh-ed25519 JGx7Ng tI6cHcYUlSikWQ38svssfxv3rvDuFZohsDO/0LFhLxM
+6pkcjuBXxEY+38JLAGKyM5i9cdp7sbgaK1c+SR4Pgsg
+-> ssh-ed25519 5SY7Kg oilxE3BBzEiS3Ufy8CIJpvZZfOXIXGFZpFbZwmGQW3Q
+mn7ILDAvu0P3CfBtsXdbMcA3SO1tCmBI2IJtU74hHYc
+-> ssh-ed25519 p/Mg4Q ZwblCBWNf3JQJAfXyW3v3VPtIqQ/noZ2UotS5Fi6Tiw
++oVJePhLAFnqzJ+reRgssQCH327L3PKe8MZnnbskxvM
+--- VLhXJtb3lZcy11wq5jj8gvY+7Ur+aLqZyQRaaEVfFlA
+Ôð¤ç·%g&J@¢bYÂ
+›ß¯éÛ<EFBFBD>ÂvéÀ_cz5V˜Á¥­'Ã­Hú„ñõßAÌÞØŒo«·
]~S‚”Ôäé•!†«]›@ gX@G“ŠYúž¸õÖ.Þ·üÜ!<21>ÿó<0B>,9†»”Ã4l\6Xbí:…Ž3—
Ì@;è9-ˆ<Y´N[·ÍÑy=½ü›ˆýùT¦­e‡9d’‘±Íu(Y:¨	

.credentials/secrets.nix

@@ -0,0 +1,6 @@
+let
+  keys = import ../keys;
+in
+{
+  "admin-environment.age".publicKeys = keys.rootKeys;
+}

.gitignore

@@ -9,3 +9,6 @@ result-*
 *.qcow2
 .gcroots
 .pre-commit-config.yaml
+
+# Ignore Terraform configuration file
+config.tf.json

default.nix

@@ -67,9 +67,18 @@ let
       commitizen.enable = true;
     };
   };
+
+  terranixConfig = import "${sources.terranix}/core" {
+    inherit pkgs;
+    strip_nulls = true;
+    terranix_config.imports = [ ./terranix ];
+  };
+  terranixConfigFile = (pkgs.formats.json { }).generate "config.tf.json" terranixConfig.config;
 in
 
 {
+  inherit terranixConfigFile terranixConfig;
+
   nodes = builtins.mapAttrs (
     host: { site, ... }: "${host}.${site}.infra.dgnum.eu"
   ) (import ./meta/nodes.nix);
@@ -83,11 +92,36 @@ in
       name = "dgnum-infra";
 
       packages = [
+        (pkgs.writeShellScriptBin "tf" ''
+          set -eo pipefail
+          ln -snf ${terranixConfigFile} config.tf.json
+          exec ${pkgs.lib.getExe pkgs.opentofu} "$@"
+        '')
+        (pkgs.writeShellScriptBin "decryptAndSourceEnvironment" ''
+          set -eo pipefail
+
+          # TODO: don't hardcode me.
+          SECRET_FILE=".credentials/admin-environment.age"
+          IDENTITIES=()
+          for identity in [ "$HOME/.ssh/id_ed25519" "$HOME/.ssh/id_rsa" ]; do
+            test -r "$identity" || continue
+            IDENTITIES+=(-i)
+            IDENTITIES+=("$identity")
+          done
+
+          test "''${#IDENTITIES[@]}" -eq 0 && echo "[agenix-shell] WARNING: no readable identities found!"
+
+          test -f "$SECRET_FILE" || echo "[agenix-shell] WARNING: encrypted environment file $SECRET_FILE not found!"
+          export eval $(${pkgs.lib.getExe pkgs.rage} --decrypt "''${IDENTITIES[@]}" -o - $SECRET_FILE)
+
+          echo "[agenix-shell] Repository-wide secrets loaded in the environment."
+        '')
         (pkgs.nixos-generators.overrideAttrs (_: {
           version = "1.8.0-unstable";
           src = builtins.storePath sources.nixos-generators;
         }))
         pkgs.npins
+        pkgs.rage
 
         (pkgs.callPackage ./lib/colmena { inherit (nix-pkgs) colmena; })
         (pkgs.callPackage "${sources.agenix}/pkgs/agenix.nix" { })
@@ -97,6 +131,8 @@ in
 
       shellHook = ''
         ${git-checks.shellHook}
+        # If we want to export these environments, we need to source it, not call it.
+        source $(which decryptAndSourceEnvironment)
       '';
 
       preferLocalBuild = true;

machines/krz01/_configuration.nix

@@ -12,11 +12,11 @@ lib.extra.mkConfig {
 
   enabledServices = [
     # INFO: This list needs to stay sorted alphabetically
+    # Machine learning API machine
+    "microvm-ml01"
     "microvm-router01"
     "nvidia-tesla-k80"
     "proxmox"
-    # Machine learning API machine
-    "microvm-ml01"
   ];
 
   extraConfig = {
@@ -57,7 +57,23 @@ lib.extra.mkConfig {
         ];
       }))
     ];
-    services.ollama = {
+
+    services = {
+      nginx = {
+        enable = true;
+        recommendedProxySettings = true;
+        virtualHosts."ollama01.beta.dgnum.eu" = {
+          enableACME = true;
+          forceSSL = true;
+          locations."/" = {
+            proxyPass = "http://${config.services.ollama.host}:${toString config.services.ollama.port}";
+            basicAuthFile = pkgs.writeText "ollama-htpasswd" ''
+              raito:$y$j9T$UDEHpLtM52hRGK0I4qT6M0$N75AhENLqgtJnTGaPzq51imhjZvuPr.ow81Co1ZTcX2
+            '';
+          };
+        };
+      };
+      ollama = {
         enable = true;
         package = pkgs.callPackage ./ollama.nix {
           cudaPackages = pkgs.cudaPackages_11;
@@ -67,5 +83,11 @@ lib.extra.mkConfig {
       };
     };
 
+    networking.firewall.allowedTCPPorts = [
+      80
+      443
+    ];
+  };
+
   root = ./.;
 }

machines/storage01/garage.nix

@@ -15,6 +15,8 @@ let
   ];
 
   buckets = [
+    "monorepo-terraform-state"
+
     "banda-website"
     "castopod-dgnum"
     "hackens-website"
@@ -77,6 +79,17 @@ in
   users.groups.garage = { };
 
   services.nginx.virtualHosts = {
+    "s3-admin.dgnum.eu" = {
+      enableACME = true;
+      forceSSL = true;
+
+      locations."/".extraConfig = ''
+        proxy_pass http://127.0.0.1:3902;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+      '';
+    };
+
     ${host} = {
       enableACME = true;
       forceSSL = true;

meta/dns.nix

@@ -87,6 +87,8 @@ let
           "*.s3"
           "cdn"
           "s3"
+          # The administration endpoint for Garage.
+          "s3-admin"
         ];
 
         rescue01.dual = [
@@ -127,6 +129,14 @@ let
           "cas-eleves"
           "vote"
         ];
+
+        krz01.dual = [
+          # Beta-grade machine learning API servers
+          "ollama01.beta"
+          "openui.beta"
+          "whisper.beta"
+          "stable-diffusion.beta"
+        ];
       }
     )
   );

meta/nodes.nix

@@ -87,6 +87,8 @@
 
     stateVersion = "24.05";
     nixpkgs = "unstable";
+
+    adminGroups = [ "lab" ];
   };
 
   storage01 = {

meta/organization.nix

@@ -55,6 +55,12 @@
       "catvayor"
       "ecoppens"
     ];
+
+    lab = [
+      "catvayor"
+      "ecoppens"
+    ];
+
   };
 
   external = {

npins/sources.json

@@ -300,6 +300,21 @@
       "url": null,
       "hash": "11vvfxw2sznc155x0xlgl00g6n9sr90xa0b1hr14vchg7gkz46r5"
     },
+    "terranix": {
+      "type": "GitRelease",
+      "repository": {
+        "type": "GitHub",
+        "owner": "terranix",
+        "repo": "terranix"
+      },
+      "pre_releases": false,
+      "version_upper_bound": null,
+      "release_prefix": null,
+      "version": "2.7.0",
+      "revision": "00710f39f38a0a654a2c4fd96cbb988b4f4cedfa",
+      "url": "https://api.github.com/repos/terranix/terranix/tarball/2.7.0",
+      "hash": "1wsyhsdsjw6xlhpkhaqvia3x0na3nx2vamcb2rbcbdmb7ra1y9f6"
+    },
     "wp4nix": {
       "type": "Git",
       "repository": {

terranix/common.nix

@@ -0,0 +1,7 @@
+{
+  # Until we get some kind of KMS operational, store secrets in the state file.
+  terraform.required_providers.secret = {
+    version = "~> 1.2.1";
+    source = "numtide/secret";
+  };
+}

terranix/default.nix

@@ -0,0 +1,7 @@
+{
+  imports = [
+    ./common.nix
+    ./state.nix
+    ./s3.nix
+  ];
+}

terranix/s3.nix

@@ -0,0 +1,32 @@
+{ lib, ... }:
+let
+  inherit (lib) tf;
+in
+{
+  # FIXME: add a NixOS module to abstract bucket creation, etc.
+  config = {
+    terraform.required_providers.garage = {
+      version = "~> 1.0.3";
+      source = "registry.opentofu.org/RaitoBezarius/garage";
+    };
+
+    resource = {
+      secret_resource.admin-s3-token.lifecycle.prevent_destroy = true;
+      garage_bucket.monorepo-terraform-state = { };
+      garage_bucket_global_alias = {
+        monorepo-terraform-state = {
+          bucket_id = tf.ref "resource.garage_bucket.monorepo-terraform-state.id";
+          alias = "monorepo-terraform-state";
+        };
+      };
+      garage_key = { };
+      garage_bucket_key = { };
+    };
+
+    provider.garage = {
+      host = "s3.dgnum.eu";
+      scheme = "https";
+      token = tf.ref "resource.secret_resource.admin-s3-token.value";
+    };
+  };
+}

terranix/state.nix

@@ -0,0 +1,21 @@
+{
+  # We use terraform.backend.s3 directly instead of the type-checked Terranix
+  # backend.s3 options. The latter does not support setting arbitrary s3
+  # endpoints.
+  #
+  # Note: currently requires the user to provide AWS_ACCESS_KEY_ID as well as
+  # AWS_SECRET_ACCESS_KEY in their environment variables.
+
+  terraform.backend.s3 = {
+    endpoints.s3 = "s3.dgnum.eu";
+    region = "garage";
+    bucket = "monorepo-terraform-state";
+    key = "state";
+
+    # It's just a dumb Garage server, don't try to be smart.
+    skip_credentials_validation = true;
+    skip_region_validation = true;
+    skip_requesting_account_id = true;
+    skip_metadata_api_check = true;
+  };
+}