feat(grfn/bbbg): Production deployment

Start of a production deployment of the app with nixos+terraform, using
provisioners and null-resources to provision nixos machines a'la espes.

Change-Id: I2ddaed76d0037dadbf9fc9e2ee27e9e67a852228
Reviewed-on: https://cl.tvl.fyi/c/depot/+/4695
Reviewed-by: grfn <grfn@gws.fyi>
Autosubmit: grfn <grfn@gws.fyi>
Tested-by: BuildkiteCI
This commit is contained in:
Griffin Smith 2021-12-26 22:37:57 -05:00 committed by clbot
parent 503ac8c782
commit 784e35bf55
9 changed files with 458 additions and 19 deletions

View file

@ -1,4 +1,4 @@
{ depot, pkgs, ... }:
args@{ depot, pkgs, ... }:
with pkgs.lib;
@ -12,6 +12,7 @@ in rec {
meta.targets = [
"db-util"
"server"
"tf"
];
depsPaths = deps.makePaths {};
@ -75,4 +76,6 @@ in rec {
server = pkgs.writeShellScriptBin "bbbg-server" ''
exec ${pkgs.openjdk17_headless}/bin/java -jar ${server-jar} "$@"
'';
tf = import ./tf.nix args;
}

View file

@ -11,6 +11,15 @@ mkShell {
openjdk11_headless
postgresql_12
nix-prefetch-git
(writeShellScriptBin "terraform" ''
set -e
module=$(nix-build ~/code/depot -A users.grfn.bbbg.tf.module)
rm -f ~/tfstate/bbbg/*.json
cp ''${module}/*.json ~/tfstate/bbbg
exec ${depot.users.grfn.bbbg.tf.terraform}/bin/terraform \
-chdir=/home/grfn/tfstate/bbbg \
"$@"
'')
];
PGHOST = "localhost";

93
users/grfn/bbbg/tf.nix Normal file
View file

@ -0,0 +1,93 @@
{ depot, ... }:
let
inherit (depot.users.grfn)
terraform
;
in terraform.workspace "bbbg" {
plugins = (p: with p; [
aws
cloudflare
]);
} {
machine = terraform.nixosMachine {
name = "bbbg";
instanceType = "t3a.small";
rootVolumeSizeGb = 250;
extraIngressPorts = [ 80 443 ];
configuration = { pkgs, lib, config, depot, ... }: {
imports = [
./module.nix
"${depot.third_party.agenix.src}/modules/age.nix"
];
services.openssh.enable = true;
services.nginx = {
enable = true;
recommendedTlsSettings = true;
recommendedOptimisation = true;
recommendedGzipSettings = true;
recommendedProxySettings = true;
};
networking.firewall.enable = false;
programs.zsh.enable = true;
users.users.grfn = {
isNormalUser = true;
initialPassword = "password";
extraGroups = [
"wheel"
"networkmanager"
"audio"
"docker"
];
shell = pkgs.zsh;
openssh.authorizedKeys.keys = [
depot.users.grfn.keys.main
];
};
security.sudo.extraRules = [{
groups = ["wheel"];
commands = [{ command = "ALL"; options = ["NOPASSWD"]; }];
}];
nix.gc = {
automatic = true;
dates = "weekly";
options = "--delete-older-than 30d";
};
age.secrets = {
bbbg.file =
depot.users.grfn.secrets."bbbg.age";
};
services.bbbg.enable = true;
services.bbbg.database.enable = true;
services.bbbg.proxy.enable = true;
services.bbbg.domain = "bbbg.gws.fyi";
security.acme.email = "root@gws.fyi";
security.acme.acceptTerms = true;
};
};
dns = {
data.cloudflare_zone.gws-fyi = {
name = "gws.fyi";
};
resource.cloudflare_record.bbbg = {
zone_id = "\${data.cloudflare_zone.gws-fyi.id}";
name = "bbbg";
type = "A";
value = "\${aws_instance.bbbg_machine.public_ip}";
proxied = false;
};
};
}

View file

@ -1,10 +1,12 @@
age-encryption.org/v1
-> ssh-ed25519 CpJBgQ dHPaZt3ZRV6rBPQrqiEpKXd48OjUC1joVIm/ZHcimVQ
Q8JwGJ91nsxspJFwZaq2BENdJYHxdHG30Ef0/Cae58M
-> ssh-ed25519 LfBFbQ oN98wLqM69Kv2Ldg31v0eBNtfpNP4nbyqAC+gCOT3yI
U8weIdIqhGs2eoKXqCxO8zHe2Ddo5fVJ5ZYua/hcBs8
-> \Z^u8-grease ., ,^=lH#0> +P=Z," d
fwUdQTFyoVYOmMUWN2nQ9JWg+Mj0iF325eJaEYkWTNvDZfUGioravnCEQxAErbAN
S1v0wgUUM8/ja3uI
--- erMVG5PLHMBECjcKtR+OLq5hYa+6dS4gPsQ5CzQByQ0
S°8÷Y×"g|DÉöZäîª0øX¶ É1¿ggïó¡ÈôÉ|¸]&½m=µ4Oô´á˜-´äéT=EmÞ8(\þb„ßïD<C3AF>¿³ ~ˆæ~+áñ“hÍÐa´~«™ReÿÃÅïØWô#Á-š5±ŽbôÉÖfO`¡mñ4ñ €<'×|U‰Ô8"<DÕõÁð2>¸<>\Ó©3$@áÏ”Ù8;Ñ|:u WKz@×%#¶ÚÇNE?Ã+!1îxNœ”“„<E2809C>¤8h>
-> ssh-ed25519 CpJBgQ 6vLlq2WEcn6TE0rgahQyl7CYhCF3uiBD3hOnZkHswmM
BSUiKPdDWMhYbi/+j9Kw5YDEOvjaickYQuhpWkhLktQ
-> ssh-ed25519 LfBFbQ mQJfyk35Ghd7UWouPlq4kTIFFwlRGh24r0kvJUgUbBw
eYpBJEG9Cdc2qHI0maFpp9/2o30R0KGLRSQ7DzsVaZ0
-> ssh-ed25519 lZtaEQ npyXpqTMWITvRVfPwEQ1rXJ0sxnJvurLOfeiE07m92E
oCXVRGOegBgQUJof8UHJsDdMyNsx6X575Rd4mWZ9LRk
-> ;^O0_l-grease
sseb4RnQz93Wlgs5B0PE+j7AzFyMkzHjFbn9sCn0UA
--- Bqq0uedob5/kJOSoavN7Aq1fH7QVNW134M3uS6u2lFA
Šr3?uÚj¾§ËF !ù_¿ÀRØ/°#B<51>ÃÁ
áN×”Ã"·òV1ÍmwºlºÍyìö's³ ˜ëÈP·å^¯6ôíK{‡ðÍÊ 3m ¬%‘Í'zOo<>éì8^SJû¤íxꨯRÞö«¤Å•zEz*Ѥú€Â>¦gë»<C3AB> %>\)Lj0ʬ«5VQ8˜/H…XOñGµ‰s7gÈæDGIÉs·Þ<4E>piuYFj?x;]

View file

@ -1,9 +1,9 @@
age-encryption.org/v1
-> ssh-ed25519 CpJBgQ w4W+pzmVIEMF0uZN7KZMAppJaLjEeDKoe7i9LGayKDQ
Rd8k+3csmbZQIrp09ZUfCAOZVwI0BZ6hCBN3nkZQMp4
-> ssh-ed25519 LfBFbQ dyv1splvcftMd1zWDkPBfsgvXxH5neZlO7ZjrhyzNHI
N/kqc/luOl8lsZcbaxF8/3ULsL78zvZhkiCarohe+G4
-> \w7t-grease lo&b JZpCA
nN2lH0W9+zulMjZMLPMk61+xsrQ
--- voTpUbu8OiJQyuKB7tIOvlErgY0jg2w7N3MehD5FIdM
&cŽz“lć Î|KśŤMŹ~<7E>®2ŕe˘şĐ¨°úUN8“Pâ~}Ý*ÇhŢÁęSYJJÇŘÁ FÉŠâoşÂc=<3D>LŹű­`z«“ŽO7€ůňËK—ÖgZ”ÝŰŐż.ÎaXD ÚHЦ€ýŮ8„7˝ŮÉĚţăó8
-> ssh-ed25519 CpJBgQ tWx7wXCFjOOfD0wKRHHvLUdR+SF0i43xvnQG9GKurnk
NRh7kSn7wqw80Y9EFr9Ccft+zYMadXZhYNPEaQlQXtQ
-> ssh-ed25519 LfBFbQ SPQMLC3Ehw00IG1CcbcLFZI2tHy89fjRgVgH4Iw2iBM
oo2gT9472/DFRoZ6TYxhnM9ylRUNzoS8mLQYvn+4OSM
-> D[7+*-grease `>j ~Jk Dz%o vaKET3
TkKVm8IpqfiVzETAi9+zuUtCdkReB+lHtthwNw
--- 3iOmY4TNICMi/Fz7k8pmoZlFym9uQBWNtHNlizoAMaM
ZPzQ6ÊäžÂ5¾½¼ATÀ ¸I¯·©;¾;Зy5]­œkÿ^!¶`¬ƒt™—$ÿRÖ<C396>tÞëK)ò<><Ýã§k¾ûû_°#XmASŠpU1©üŽ@šÂ)ûâc©ÖºÝqœÿj1zÑ,HÄÂÞgÈß:è‘

View file

@ -1,9 +1,10 @@
let
grfn = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMcBGBoWd5pPIIQQP52rcFOQN3wAY0J/+K2fuU6SffjA";
mugwump = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFE2fxPgWO+zeQoLBTgsgxP7Vg7QNHlrQ+Rb3fHFTomB";
bbbg = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL/VzrNEY47KPTce3dgfORkAbweWkr4BI8j54BAIs7bG";
in
{
"bbbg.age".publicKeys = [ grfn mugwump ];
"bbbg.age".publicKeys = [ grfn mugwump bbbg ];
"cloudflare.age".publicKeys = [ grfn mugwump ];
}

View file

@ -0,0 +1,24 @@
{ pkgs, ... }:
{
provider.aws = map (region: {
inherit region;
alias = region;
profile = "personal";
}) [
"us-east-1"
"us-east-2"
"us-west-2"
];
data.external.cloudflare_api_key = {
program = [(pkgs.writeShellScript "cloudflare_api_key" ''
jq -n --arg api_key "$(pass cloudflare-api-key)" '{"api_key":$api_key}'
'')];
};
provider.cloudflare = {
email = "root@gws.fyi";
api_key = "\${data.external.cloudflare_api_key.result.api_key}";
};
}

View file

@ -0,0 +1,203 @@
{ depot, pkgs, lib, ... }:
# mostly stolen from espes
{ name
, instanceType
, configuration
, prefix ? "${name}_"
, region ? "us-east-2"
, rootVolumeSizeGb ? 50
, securityGroupId ? null
, extraIngressPorts ? []
}:
let
os = depot.ops.nixos.nixosFor ({ modulesPath, ... }: {
imports = [
"${pkgs.path}/nixos/modules/virtualisation/amazon-image.nix"
configuration
];
ec2.hvm = true;
networking.hostName = name;
# TODO: remove this once the terraform tls provider supports ed25519 keys
# https://github.com/hashicorp/terraform-provider-tls/issues/26
services.openssh.extraConfig = ''
PubkeyAcceptedKeyTypes=+ssh-rsa
PubkeyAcceptedAlgorithms=+ssh-rsa
'';
});
targetUser = "root";
ec2Amis = import "${pkgs.path}/nixos/modules/virtualisation/ec2-amis.nix";
osRoot = os.config.system.build.toplevel;
osRootPath = builtins.unsafeDiscardStringContext (toString osRoot.outPath);
drvPath = builtins.unsafeDiscardStringContext (toString osRoot.drvPath);
machineResource = "aws_instance.${prefix}machine";
recursiveMerge = builtins.foldl' lib.recursiveUpdate {};
securityGroupId' =
if isNull securityGroupId
then "\${aws_security_group.${prefix}group.id}"
else securityGroupId;
in recursiveMerge [
(lib.optionalAttrs (isNull securityGroupId) {
resource.aws_security_group."${prefix}group" = {
provider = "aws.${region}";
vpc_id = null;
# terraform isn't good about knowing what other resources depend on
# security groups
lifecycle.create_before_destroy = true;
};
resource.aws_security_group_rule.all_egress = {
provider = "aws.${region}";
security_group_id = securityGroupId';
type = "egress";
protocol = "-1";
from_port = 0;
to_port = 0;
cidr_blocks = ["0.0.0.0/0"];
ipv6_cidr_blocks = ["::/0"];
description = null;
prefix_list_ids = null;
self = null;
};
})
rec {
data.external.my_ip = {
program = [(pkgs.writeShellScript "my_ip" ''
${pkgs.jq}/bin/jq \
-n \
--arg ip "$(curl ifconfig.me)" \
'{"ip":$ip}'
'')];
};
resource.aws_security_group_rule.provision_ssh_access = {
provider = "aws.${region}";
security_group_id = securityGroupId';
type = "ingress";
protocol = "TCP";
from_port = 22;
to_port = 22;
cidr_blocks = ["\${data.external.my_ip.result.ip}/32"];
ipv6_cidr_blocks = [];
description = null;
prefix_list_ids = null;
self = null;
};
resource.tls_private_key."${prefix}key" = {
algorithm = "RSA";
};
resource.aws_key_pair."${prefix}generated_key" = {
provider = "aws.${region}";
key_name = "generated-key-\${sha256(tls_private_key.${prefix}key.public_key_openssh)}";
public_key = "\${tls_private_key.${prefix}key.public_key_openssh}";
};
resource.aws_instance."${prefix}machine" = {
provider = "aws.${region}";
ami = ec2Amis."21.05"."${region}".hvm-ebs;
instance_type = instanceType;
vpc_security_group_ids = [ securityGroupId' ];
key_name = "\${aws_key_pair.${prefix}generated_key.key_name}";
root_block_device = {
volume_size = rootVolumeSizeGb;
tags.Name = name;
};
tags.Name = name;
};
resource.null_resource."${prefix}deploy_nixos" = {
triggers = {
# deploy if the machine is recreated
machine_id = "\${${machineResource}.id}";
# deploy on os changes
os_drv = drvPath;
};
connection = {
type = "ssh";
host = "\${${machineResource}.public_ip}";
user = targetUser;
private_key = "\${tls_private_key.${prefix}key.private_key_pem}";
};
# do the actual deployment
provisioner = [
# wait till ssh is up
{ remote-exec.inline = [ "true" ]; }
# copy the nixos closure
{
local-exec.command = ''
export PATH="${pkgs.openssh}/bin:$PATH"
scratch="$(mktemp -d)"
trap 'rm -rf -- "$scratch"' EXIT
# write out ssh key
echo -n "''${tls_private_key.${prefix}key.private_key_pem}" > $scratch/id_rsa.pem
chmod 0600 $scratch/id_rsa.pem
export NIX_SSHOPTS="\
-o StrictHostKeyChecking=no\
-o UserKnownHostsFile=/dev/null\
-o GlobalKnownHostsFile=/dev/null\
-o IdentityFile=$scratch/id_rsa.pem"
nix-build ${drvPath}
nix-copy-closure \
--to ${targetUser}@''${${machineResource}.public_ip} \
${osRootPath} \
--gzip \
--use-substitutes
'';
}
# activate it
{
remote-exec.inline = [
# semicolons mandatory
''
set -e;
nix-env --profile /nix/var/nix/profiles/system --set ${osRootPath};
${osRootPath}/bin/switch-to-configuration switch;
''
];
}
];
};
}
{
resource.aws_security_group_rule = builtins.listToAttrs (map (port: {
name = "ingress_${toString port}";
value = {
provider = "aws.${region}";
security_group_id = securityGroupId';
type = "ingress";
protocol = "TCP";
from_port = port;
to_port = port;
cidr_blocks = ["0.0.0.0/0"];
ipv6_cidr_blocks = [];
description = null;
prefix_list_ids = null;
self = null;
};
}) extraIngressPorts);
}
]

View file

@ -0,0 +1,104 @@
{ pkgs, depot, ... }:
name: { plugins }: module_tf:
let
inherit (pkgs) lib runCommandNoCC writeText writeScript;
inherit (lib) filterAttrsRecursive;
allPlugins = (p: plugins p ++ (with p; [
external
local
tls
p.null
]));
tf = pkgs.terraform.withPlugins allPlugins;
cleanTerraform = filterAttrsRecursive (k: _: ! (builtins.elem k [
"__readTree"
"__readTreeChildren"
]));
plugins_tf = {
terraform.required_providers = (builtins.listToAttrs (map (p: {
name = lib.last (lib.splitString "/" p.provider-source-address);
value = {
source = p.provider-source-address;
version = p.version;
};
}) (allPlugins pkgs.terraform.plugins)));
};
module_tf' = module_tf // {
inherit (depot.users.grfn.terraform) globals;
plugins = plugins_tf;
};
module = runCommandNoCC "module" {} ''
mkdir $out
${lib.concatStrings (lib.mapAttrsToList (k: config_tf:
(let
# TODO: filterAttrsRecursive?
configJson = writeText "${k}.tf.json"
(builtins.toJSON (cleanTerraform config_tf));
in ''
${pkgs.jq}/bin/jq . ${configJson} > $out/${lib.escapeShellArg k}.tf.json
''))
(cleanTerraform module_tf'))}
'';
tfcmd = writeScript "${name}-tfcmd" ''
set -e
dir="''${TF_STATE_ROOT:-$HOME/tfstate}/${name}"
cd "$dir"
rm -f *.json
cp ${module}/*.json .
exec ${tf}/bin/terraform "$(basename "$0")"
'';
init = writeScript "${name}-init" ''
set -e
dir="''${TF_STATE_ROOT:-$HOME/tfstate}/${name}"
[ -d "$dir" ] || mkdir -p "$dir"
cd "$dir"
rm -f *.json
cp ${module}/*.json .
exec ${tf}/bin/terraform init
'';
# TODO: import (-config)
tfcmds = runCommandNoCC "${name}-tfcmds" {} ''
mkdir -p $out/bin
ln -s ${init} $out/bin/init
ln -s ${tfcmd} $out/bin/validate
ln -s ${tfcmd} $out/bin/plan
ln -s ${tfcmd} $out/bin/apply
ln -s ${tfcmd} $out/bin/destroy
'';
in {
inherit name module;
terraform = tf;
cmds = tfcmds;
# run = {
# init = depot.nix.nixRunWrapper "init" tfcmds;
# validate = depot.nix.nixRunWrapper "validate" tfcmds;
# plan = depot.nix.nixRunWrapper "plan" tfcmds;
# apply = depot.nix.nixRunWrapper "apply" tfcmds;
# destroy = depot.nix.nixRunWrapper "destroy" tfcmds;
# };
test = runCommandNoCC "${name}-test" {} ''
set -e
export TF_STATE_ROOT=$(pwd)
${tfcmds}/bin/init
${tfcmds}/bin/validate
touch $out
'';
meta.targets = [ "module" "test" ];
}