From 8706116ce4141d91da90b7494cf172d952bc90f3 Mon Sep 17 00:00:00 2001 From: Ryan Lahfa Date: Thu, 10 Oct 2024 12:27:40 +0200 Subject: [PATCH 1/3] feat(infra): introduce Terranix This requires the support for monorepo-terraform-state.s3.dgnum.eu being available. `.credentials/` is age-encrypted using only my key for now until we figure out the right mechanism. Signed-off-by: Ryan Lahfa --- .credentials/admin-environment.age | Bin 0 -> 1487 bytes .credentials/secrets.nix | 6 +++++ .gitignore | 6 +++++ .terraform.lock.hcl | 38 +++++++++++++++++++++++++++++ default.nix | 36 +++++++++++++++++++++++++++ npins/sources.json | 15 ++++++++++++ terranix/common.nix | 7 ++++++ terranix/default.nix | 6 +++++ terranix/state.nix | 21 ++++++++++++++++ 9 files changed, 135 insertions(+) create mode 100644 .credentials/admin-environment.age create mode 100644 .credentials/secrets.nix create mode 100644 .terraform.lock.hcl create mode 100644 terranix/common.nix create mode 100644 terranix/default.nix create mode 100644 terranix/state.nix diff --git a/.credentials/admin-environment.age b/.credentials/admin-environment.age new file mode 100644 index 0000000000000000000000000000000000000000..bcd8f7ed8ac579bd82e9ca22cfcf6590c5a3e138 GIT binary patch literal 1487 zcmZXU&Fk9)7{^6KU153{gMkN&hv^7EnbBw-J^K*2)`N(YkYY+R|uZSBnFf}l}@oprs83~5eoku{36T~THA z&d^4r9whnJK!7=aIGf@za*&n+5!aHM8Xsdux2Ph5AEq|na|$Hh0Ngx9io~J2DWhfy zsC7U%avDTuTyPfL8>OU{&f(CIXd5?038nH}#N#0^K#CC))AdrlMN6|;Gf4IRNNR}G zFfuXSg&k>45ObfMGXuDY(m=s_)OJL*#wMNl$$%wMyS81MPkV?*@I8j{Ni^T)jRm07 zv;}l!DE7C|CSTZ5*ybcy^_2E}sud2&3M(%0`b56LyuhdkZ;2{+ZUSR&6&G>?t8^oB zi8zP0M#WX&(8HqAjpyWo9QQW}n$SDopvBTuT@F}1XZ>y)rZQ$_`$gFH$c9XE^VL>I zU~e**a;0YWJS~<+T8-L*O#<{x1X+&>7|ma8>i{+heW;-@FPpjpApC%7kU&Y>CMvT4 z+m$Ji@j<8qSVDis_?uN*9RjTt<8@0vher0IS9ZF%>@Sc6!gw}A2&6r%Mf zEZpmhIu)*K=hn4v^|n=Rq=nR^mueTC`)iR6>h^9>CXBca1Stg0lRcl9RSN>LLsXmY znuJBT7E39|0htpIEmw37p{F|1D7y01zf~e$5LQ zE1SE7**=Lv^L7=LQ<|wf!p@;#HrX%@-Bb#Tv1c1Nh!aZ}3qB2<%*&I+2tg`SD$oSO z7%;SiI=CAXRKRKNySu87uP zt>{C%y~_F<9EAd7IuJn>>mq_OQ`*p<2##TBx(Ws3p=`LYG2n$1bT%HjOE@ommtcOJgpU zbyY`p{&yu2TGVVr4a8(4T7j>t%R1gUxwqFGG9d!(R86D+hLad2SX)HYRI(sODDxcV z6W=O#uVlnvjo;f$?G3Jb@q?5^hcNOeCFg^#$jCk`LP?9uI(Q| zzWn$7xBnvePx5m|uibGOpy|pv@7z4{`inQs!!MjTcWV0MvmabIl~?aL4>im5#)i>^y rFJFIt@b{VM+7A!j^T_b#m5*OJ@y5x`yKnN}+;!dl_4L!ne){e|dieP7 literal 0 HcmV?d00001 diff --git a/.credentials/secrets.nix b/.credentials/secrets.nix new file mode 100644 index 0000000..c899d92 --- /dev/null +++ b/.credentials/secrets.nix @@ -0,0 +1,6 @@ +let + keys = import ../keys; +in +{ + "admin-environment.age".publicKeys = keys.rootKeys; +} diff --git a/.gitignore b/.gitignore index d50eeb8..d85129f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,9 @@ result-* *.qcow2 .gcroots .pre-commit-config.yaml + +# Ignore Terraform configuration file +config.tf.json + +# Ignore Terraform stuff +.terraform diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl new file mode 100644 index 0000000..2e96052 --- /dev/null +++ b/.terraform.lock.hcl @@ -0,0 +1,38 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/numtide/secret" { + version = "1.2.1" + constraints = "~> 1.2.1" + hashes = [ + "h1:t2z3CjxVsXjKb3g59WGkLtvDIR4NzLU7UFEcyAgF2C0=", + "zh:17cbc7f3b90ee2b3ae5adfc3bd9cb70166a5ffbd8e642e64afa7cb0e32a34bae", + "zh:5d66ce2aea25fc3c12cec6fc569b8ff314df6d773b9c3449983a4e9cde8347c7", + "zh:67d02e96bf0d07f2fcf16ce9427a7a26f53e695676405d0c2b815808f950411d", + "zh:77c3c05681ce199e6b0e2e5a2dfe418f61ae8863d527e7a7d47a9699d912683b", + "zh:7f37e633b4f94ba9f347cfe68d44f80fe066188feb954b13ee0f621caae4121d", + "zh:ea16bbe494c6ddd0af7bbea9554474c387517db4e7f0d15513bb29ff893871bc", + ] +} + +provider "registry.opentofu.org/raitobezarius/garage" { + version = "1.0.3" + constraints = "~> 1.0.3" + hashes = [ + "h1:QKbZcU7u9OG1t/h4S3+pXS3sOUfVMmfLTiYh5L5j1rE=", + "zh:04f220a2baf4bd1bae07888a1c311cacd6076c209de83adbe573525fc50f2ea4", + "zh:078938d5fa07e024d779c664823427af28935bbeb77e0ff940bac3e7bc41f1e8", + "zh:2dd58a2d82094a1b07ff1b6de57e4a0d96e1f20abecd4f70a6469079b46b76d9", + "zh:325da7a74b1c84f934b38134d7c419253292aeed6f6836a2fb37f42d13a8ff67", + "zh:3ca9230ef87e70691b24fd83d40bb5b6a08f0b91ab26cbb2e692f92155b6d179", + "zh:45ef683a18a5053c93c691d08f3903fd4918467dfa056b1c274207de8a6aeb74", + "zh:4c9ee6c34b07c209c5daf1e9ff182f828667e54a90a683bc11cdcea86e4f8ef7", + "zh:5f0bb6524b2fffa606e0e3585af93dfc31b611c7abf55e4371ae5fc36e85972c", + "zh:7a3495dc211164c7d4042769c20d7111c767d0fd5908742e0766281c70d7d184", + "zh:7ce79867cdd4b1f7028da811cd5cb271a46820c79c0328a1221dd3bb6215c631", + "zh:93278861ee6bcb64e23bd1268f79b02035fba4fca0a98607a98f46abf8dfdf83", + "zh:937e681beea8b0dd899557f2a194c8128bd8810417ff04954bc9958ff826e980", + "zh:cae6e1598dd32f23f3900c41e50a6ece7d9456dbd033d855bb238ac21539d67b", + "zh:f6f7556ba7d5578604290170a709e00140be6d7f8a510a20bce49a9a23d75e5f", + ] +} diff --git a/default.nix b/default.nix index 414feb8..cb1ec95 100644 --- a/default.nix +++ b/default.nix @@ -67,9 +67,18 @@ let commitizen.enable = true; }; }; + + terranixConfig = import "${sources.terranix}/core" { + inherit pkgs; + strip_nulls = true; + terranix_config.imports = [ ./terranix ]; + }; + terranixConfigFile = (pkgs.formats.json { }).generate "config.tf.json" terranixConfig.config; in { + inherit terranixConfigFile terranixConfig; + nodes = builtins.mapAttrs ( host: { site, ... }: "${host}.${site}.infra.dgnum.eu" ) (import ./meta/nodes.nix); @@ -83,11 +92,36 @@ in name = "dgnum-infra"; packages = [ + (pkgs.writeShellScriptBin "tf" '' + set -eo pipefail + ln -snf ${terranixConfigFile} config.tf.json + exec ${pkgs.lib.getExe pkgs.opentofu} "$@" + '') + (pkgs.writeShellScriptBin "decryptAndSourceEnvironment" '' + set -eo pipefail + + # TODO: don't hardcode me. + SECRET_FILE=".credentials/admin-environment.age" + IDENTITIES=() + for identity in [ "$HOME/.ssh/id_ed25519" "$HOME/.ssh/id_rsa" ]; do + test -r "$identity" || continue + IDENTITIES+=(-i) + IDENTITIES+=("$identity") + done + + test "''${#IDENTITIES[@]}" -eq 0 && echo "[agenix-shell] WARNING: no readable identities found!" + + test -f "$SECRET_FILE" || echo "[agenix-shell] WARNING: encrypted environment file $SECRET_FILE not found!" + export eval $(${pkgs.lib.getExe pkgs.rage} --decrypt "''${IDENTITIES[@]}" -o - $SECRET_FILE) + + echo "[agenix-shell] Repository-wide secrets loaded in the environment." + '') (pkgs.nixos-generators.overrideAttrs (_: { version = "1.8.0-unstable"; src = builtins.storePath sources.nixos-generators; })) pkgs.npins + pkgs.rage (pkgs.callPackage ./lib/colmena { inherit (nix-pkgs) colmena; }) (pkgs.callPackage "${sources.agenix}/pkgs/agenix.nix" { }) @@ -97,6 +131,8 @@ in shellHook = '' ${git-checks.shellHook} + # If we want to export these environments, we need to source it, not call it. + source $(which decryptAndSourceEnvironment) ''; preferLocalBuild = true; diff --git a/npins/sources.json b/npins/sources.json index 7a1c44f..8834751 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -294,6 +294,21 @@ "url": null, "hash": "166057469hhxnyqbpd7jjlccdmigzch51616n1d5r617xg0y1mwp" }, + "terranix": { + "type": "GitRelease", + "repository": { + "type": "GitHub", + "owner": "terranix", + "repo": "terranix" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, + "version": "2.7.0", + "revision": "00710f39f38a0a654a2c4fd96cbb988b4f4cedfa", + "url": "https://api.github.com/repos/terranix/terranix/tarball/2.7.0", + "hash": "1wsyhsdsjw6xlhpkhaqvia3x0na3nx2vamcb2rbcbdmb7ra1y9f6" + }, "wp4nix": { "type": "Git", "repository": { diff --git a/terranix/common.nix b/terranix/common.nix new file mode 100644 index 0000000..0e8b0be --- /dev/null +++ b/terranix/common.nix @@ -0,0 +1,7 @@ +{ + # Until we get some kind of KMS operational, store secrets in the state file. + terraform.required_providers.secret = { + version = "~> 1.2.1"; + source = "numtide/secret"; + }; +} diff --git a/terranix/default.nix b/terranix/default.nix new file mode 100644 index 0000000..b6ff81e --- /dev/null +++ b/terranix/default.nix @@ -0,0 +1,6 @@ +{ + imports = [ + ./common.nix + ./state.nix + ]; +} diff --git a/terranix/state.nix b/terranix/state.nix new file mode 100644 index 0000000..ef2f7d5 --- /dev/null +++ b/terranix/state.nix @@ -0,0 +1,21 @@ +{ + # We use terraform.backend.s3 directly instead of the type-checked Terranix + # backend.s3 options. The latter does not support setting arbitrary s3 + # endpoints. + # + # Note: currently requires the user to provide AWS_ACCESS_KEY_ID as well as + # AWS_SECRET_ACCESS_KEY in their environment variables. + + terraform.backend.s3 = { + endpoints.s3 = "s3.dgnum.eu"; + region = "garage"; + bucket = "monorepo-terraform-state"; + key = "state"; + + # It's just a dumb Garage server, don't try to be smart. + skip_credentials_validation = true; + skip_region_validation = true; + skip_requesting_account_id = true; + skip_metadata_api_check = true; + }; +} -- 2.47.0 From 33f7a082ce80b9317cad5f7160cd35d1eb522621 Mon Sep 17 00:00:00 2001 From: Ryan Lahfa Date: Thu, 10 Oct 2024 12:31:44 +0200 Subject: [PATCH 2/3] feat(infra): add S3 declarative buckets A very simple basic support for it, which requires a S3 admin token. Signed-off-by: Ryan Lahfa --- terranix/default.nix | 1 + terranix/s3.nix | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 terranix/s3.nix diff --git a/terranix/default.nix b/terranix/default.nix index b6ff81e..bcf5704 100644 --- a/terranix/default.nix +++ b/terranix/default.nix @@ -2,5 +2,6 @@ imports = [ ./common.nix ./state.nix + ./s3.nix ]; } diff --git a/terranix/s3.nix b/terranix/s3.nix new file mode 100644 index 0000000..5255d6b --- /dev/null +++ b/terranix/s3.nix @@ -0,0 +1,32 @@ +{ lib, ... }: +let + inherit (lib) tf; +in +{ + # FIXME: add a NixOS module to abstract bucket creation, etc. + config = { + terraform.required_providers.garage = { + version = "~> 1.0.3"; + source = "registry.opentofu.org/RaitoBezarius/garage"; + }; + + resource = { + secret_resource.admin-s3-token.lifecycle.prevent_destroy = true; + garage_bucket.monorepo-terraform-state = { }; + garage_bucket_global_alias = { + monorepo-terraform-state = { + bucket_id = tf.ref "resource.garage_bucket.monorepo-terraform-state.id"; + alias = "monorepo-terraform-state"; + }; + }; + garage_key = { }; + garage_bucket_key = { }; + }; + + provider.garage = { + host = "s3-admin.dgnum.eu"; + scheme = "https"; + token = tf.ref "resource.secret_resource.admin-s3-token.value"; + }; + }; +} -- 2.47.0 From 822b4f3b833ac5f74374f1931141422c8d425c23 Mon Sep 17 00:00:00 2001 From: Ryan Lahfa Date: Thu, 10 Oct 2024 17:51:57 +0200 Subject: [PATCH 3/3] feat(infra): showcase the declarative bucket feature Signed-off-by: Ryan Lahfa --- terranix/s3.nix | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/terranix/s3.nix b/terranix/s3.nix index 5255d6b..94bb6c5 100644 --- a/terranix/s3.nix +++ b/terranix/s3.nix @@ -12,15 +12,37 @@ in resource = { secret_resource.admin-s3-token.lifecycle.prevent_destroy = true; - garage_bucket.monorepo-terraform-state = { }; + garage_bucket = { + monorepo-terraform-state = { }; + impress-raito-demo = { }; + }; + garage_bucket_global_alias = { monorepo-terraform-state = { bucket_id = tf.ref "resource.garage_bucket.monorepo-terraform-state.id"; alias = "monorepo-terraform-state"; }; + impress-raito-demo = { + bucket_id = tf.ref "resource.garage_bucket.impress-raito-demo.id"; + alias = "impress-raito-demo"; + }; + }; + + garage_key = { + raito-dinum-test = { + name = "raito-dinum-test"; + permissions.create_bucket = false; + }; + }; + garage_bucket_key = { + raito-dinum-test = { + bucket_id = tf.ref "resource.garage_bucket.impress-raito-demo.id"; + access_key_id = tf.ref "resource.garage_key.raito-dinum-test.access_key_id"; + read = true; + write = true; + owner = true; + }; }; - garage_key = { }; - garage_bucket_key = { }; }; provider.garage = { -- 2.47.0