Merge pull request #9 from tazjin/feat/read-tree

Configure automatic package layouts via repository structure
This commit is contained in:
Vincent Ambo 2019-11-15 23:46:13 +00:00 committed by GitHub
commit ae53bf30c3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 186 additions and 106 deletions

View file

@ -14,88 +14,44 @@ let
url = "https://github.com/NixOS/nixpkgs-channels/archive/${stableCommit}.tar.gz";
sha256 = "0243qiivxl3z51biy4f5y5cy81x5bki5dazl9wqwgnmd373gpmxy";
};
readTree = import ./read-tree.nix;
unstableCommit = "765a71f15025ce78024bae3dc4a92bd2be3a8fbf";
unstableSrc = fetchTarball {
url = "https://github.com/NixOS/nixpkgs-channels/archive/${unstableCommit}.tar.gz";
sha256 = "0j1wghr9dj9njn3x9xi0wzjk1107gi2pxb0w2dk8g0djmhnlx71q";
};
unstable = import unstableSrc {};
# Derivations that have `meta.enableCI` set to `true` should be
# built by the CI system on every commit. This code implements
# filtering of all derivations in the local sets against this
# condition.
filterCI = lib: pkgs: let
inherit (lib) collect isDerivation filterAttrsRecursive;
ciCondition = _: x: (!isDerivation x) || ((x ? meta.enableCI) && (x.meta.enableCI));
in collect isDerivation (filterAttrsRecursive ciCondition pkgs);
localPkgs = self: super: {
# Local projects should be added here:
tazjin = {
blog = self.callPackage ./services/tazblog {};
blog_cli = self.callPackage ./tools/blog_cli {};
gemma = self.callPackage ./services/gemma {};
nixcon = self.naersk.buildPackage ./services/nixcon-demo {};
repoPkgs = self: super:
let config = {
pkgs = self;
upstream = super;
kms_pass = self.callPackage ./tools/kms_pass {
kms = {
project = "tazjins-infrastructure";
region = "europe-north1";
keyring = "tazjins-keys";
key = "kontemplate-key";
};
};
# Third-party projects (either vendored or modified from nixpkgs) go here:
nixery = import ./third_party/nixery.nix { pkgs = self; };
terraform-gcp = self.terraform_0_12.withPlugins(p: [ p.google p.google-beta ]);
ormolu = import (self.fetchFromGitHub {
owner = "tweag";
repo = "ormolu";
rev = "a7076c0f83e5c06ea9067b71171859fa2ba8afd9";
sha256 = "1p4n2ja4ciw3qfskn65ggpy37mvgf2sslxqmqn8s8jjarnqcyfny";
}) { pkgs = self; };
naersk = self.callPackage (self.fetchFromGitHub {
owner = "nmattia";
repo = "naersk";
rev = "68c1c2b2b661913cdc5ecabea518dfdc4f449027";
sha256 = "1ll310pl44kdbwfslzwvg2v7khf1y0xkg2j5wcfia4k7sj6bcl28";
}) {};
# Gemma needs an older version of Elm to be built. Updating it to
# the newer version is a lot of effort.
elmPackages = (import (self.fetchFromGitHub {
owner = "NixOS";
repo = "nixpkgs";
rev = "14f9ee66e63077539252f8b4550049381a082518";
sha256 = "1wn7nmb1cqfk2j91l3rwc6yhimfkzxprb8wknw5wi57yhq9m6lv1";
}) {}).elmPackages;
# Wrap kontemplate to inject the Cloud KMS version of 'pass'
kontemplate =
let master = super.kontemplate.overrideAttrs(_: {
src = self.fetchFromGitHub {
owner = "tazjin";
repo = "kontemplate";
rev = "v1.8.0";
sha256 = "123mjmmm4hynraq1fpn3j5i0a1i87l265kkjraxxxbl0zacv74i1";
};
});
in self.writeShellScriptBin "kontemplate" ''
export PATH="${self.tazjin.kms_pass}/bin:$PATH"
exec ${master}/bin/kontemplate $@
'';
# One of Gemma's dependencies is missing in nixpkgs' Quicklisp
# package set, it is overlaid locally here.
lispPackages = import ./third_party/common_lisp/quicklisp.nix {
inherit (self) lib;
inherit (super) lispPackages;
in {
services = readTree ./services config;
tools = readTree ./tools config;
third_party = readTree ./third_party config;
}
# Load overrides into the top-level:
// (readTree ./overrides config)
# Collect all projects that should be built by CI
// {
ciProjects = (filterCI super.lib self.services)
++ (filterCI super.lib self.tools)
++ (filterCI super.lib self.third_party);
};
# All projects that should be built by CI should be added here:
ciProjects = [
self.kontemplate
self.nixery
self.ormolu
self.terraform-gcp
] ++ filter (d: d ? meta.broken && !d.meta.broken) (attrValues self.tazjin);
};
in { ... } @ args: import stableSrc (args // {
overlays = [ localPkgs ];
overlays = [ repoPkgs ];
config.allowUnfree = true;
config.allowBroken = true;
})

View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: tazblog
image: nixery.local/shell/tazjin.blog:{{ gitHEAD }}
image: nixery.local/shell/services.tazblog:{{ gitHEAD }}
command: [ "tazblog" ]
---
apiVersion: v1

10
overrides/elmPackages.nix Normal file
View file

@ -0,0 +1,10 @@
# Gemma needs an older version of Elm to be built. Updating it to
# the newer version is a lot of effort.
{ pkgs, ... }:
(import (pkgs.fetchFromGitHub {
owner = "NixOS";
repo = "nixpkgs";
rev = "14f9ee66e63077539252f8b4550049381a082518";
sha256 = "1wn7nmb1cqfk2j91l3rwc6yhimfkzxprb8wknw5wi57yhq9m6lv1";
}) {}).elmPackages

14
overrides/kontemplate.nix Normal file
View file

@ -0,0 +1,14 @@
{ pkgs, upstream, ... }:
let master = upstream.kontemplate.overrideAttrs(_: {
src = pkgs.fetchFromGitHub {
owner = "tazjin";
repo = "kontemplate";
rev = "v1.8.0";
sha256 = "123mjmmm4hynraq1fpn3j5i0a1i87l265kkjraxxxbl0zacv74i1";
};
});
in pkgs.writeShellScriptBin "kontemplate" ''
export PATH="${pkgs.tools.kms_pass}/bin:$PATH"
exec ${master}/bin/kontemplate $@
''

View file

@ -0,0 +1,8 @@
# One of Gemma's dependencies is missing in nixpkgs' Quicklisp
# package set, it is overlaid locally here.
{ pkgs, upstream, ... }:
import ./quicklisp.nix {
inherit (pkgs) lib;
inherit (upstream) lispPackages;
}

View file

@ -1,6 +1,3 @@
# Overlay over `pkgs.lispPackages` that adds additional packages which
# are missing from the imported Quicklisp package set in nixpkgs.
{ lib, lispPackages }:
let inherit (lispPackages) buildLispPackage qlOverrides fetchurl;

65
read-tree.nix Normal file
View file

@ -0,0 +1,65 @@
path: { pkgs, ... } @ args:
let
inherit (builtins)
attrNames
attrValues
filter
head
isString
listToAttrs
map
match
readDir
tail
toPath
toString;
zipAttrs = names: values:
if (names == []) || (values == [])
then []
else [{
name = head names;
value = head values;
}] ++ zipAttrs (tail names) (tail values);
attrsToList = attrs: zipAttrs (attrNames attrs) (attrValues attrs);
isFile = s: s == "regular";
isDir = s: s == "directory";
joinPath = p: f: toPath ((toString p) + "/" + f);
isNixFile = file:
let res = match "(.*)\.nix" file;
in if res == null then null else head res;
filterNixFiles = dir:
let files = filter (e: isFile e.value) dir;
nixFiles = map (f: {
# Name and value are intentionally flipped to get the
# correct attribute set structure back out
name = isNixFile f.name;
value = f.name; # i.e. the path
}) files;
in filter (f: isString f.name) nixFiles;
traverse = path: dir:
let nixFiles = filterNixFiles dir;
imported = map (f: {
inherit (f) name;
value = import (joinPath path f.value) args;
}) nixFiles;
dirs = map (d: {
inherit (d) name;
value = readTree (joinPath path d.name);
}) (filter (e: isDir e.value) dir);
in listToAttrs (imported ++ dirs);
importOr = path: dir: f:
if dir ? "default.nix"
then import path args
else f path (attrsToList dir);
readTree = path: importOr path (readDir path) traverse;
in readTree path

View file

@ -1,17 +1,20 @@
{ stdenv, sbcl, lispPackages, elmPackages, makeWrapper, openssl }:
{ pkgs, ... }:
let frontend = stdenv.mkDerivation {
name = "gemma-frontend";
src = ./frontend;
buildInputs = [ elmPackages.elm ];
let
inherit (pkgs) stdenv sbcl lispPackages elmPackages makeWrapper openssl;
phases = [ "unpackPhase" "buildPhase" ];
buildPhase = ''
mkdir .home && export HOME="$PWD/.home"
mkdir -p $out
elm-make --yes Main.elm --output $out/index.html
'';
};
frontend = stdenv.mkDerivation {
name = "gemma-frontend";
src = ./frontend;
buildInputs = [ elmPackages.elm ];
phases = [ "unpackPhase" "buildPhase" ];
buildPhase = ''
mkdir .home && export HOME="$PWD/.home"
mkdir -p $out
elm-make --yes Main.elm --output $out/index.html
'';
};
in stdenv.mkDerivation rec {
name = "gemma";
src = ./.;

View file

@ -0,0 +1,3 @@
{ pkgs, ... }:
pkgs.third_party.naersk.buildPackage ./. {}

View file

@ -2,9 +2,10 @@
#
# tazblog.nix was generated using cabal2nix.
{ writeShellScriptBin, haskell }:
{ pkgs, ... }:
let
inherit (pkgs) writeShellScriptBin haskell;
tazblog = haskell.packages.ghc865.callPackage ./tazblog.nix {};
wrapper = writeShellScriptBin "tazblog" ''
export PORT=8000
@ -13,4 +14,5 @@ let
'';
in wrapper.overrideAttrs(_: {
allowSubstitutes = true;
meta.enableCI = true;
})

9
third_party/naersk.nix vendored Normal file
View file

@ -0,0 +1,9 @@
{ pkgs, ... }:
let inherit (pkgs) callPackage fetchFromGitHub;
in callPackage (fetchFromGitHub {
owner = "nmattia";
repo = "naersk";
rev = "68c1c2b2b661913cdc5ecabea518dfdc4f449027";
sha256 = "1ll310pl44kdbwfslzwvg2v7khf1y0xkg2j5wcfia4k7sj6bcl28";
}) {}

View file

@ -1,6 +1,6 @@
# Technically I suppose Nixery is not a third-party program, but it's
# outside of this repository ...
{ pkgs }:
{ pkgs, ... }:
let src = pkgs.fetchFromGitHub {
owner = "google";

8
third_party/ormolu.nix vendored Normal file
View file

@ -0,0 +1,8 @@
{ pkgs, ... }:
import (pkgs.fetchFromGitHub {
owner = "tweag";
repo = "ormolu";
rev = "a7076c0f83e5c06ea9067b71171859fa2ba8afd9";
sha256 = "1p4n2ja4ciw3qfskn65ggpy37mvgf2sslxqmqn8s8jjarnqcyfny";
}) { inherit pkgs; }

3
third_party/terraform-gcp.nix vendored Normal file
View file

@ -0,0 +1,3 @@
{ pkgs, ... }:
pkgs.terraform_0_12.withPlugins(p: [ p.google p.google-beta ])

View file

@ -11,19 +11,19 @@ readonly TARGET_TOOL=$(basename $0)
case "${TARGET_TOOL}" in
terraform)
attr="terraform-gcp"
attr="third_party.terraform-gcp"
;;
kontemplate)
attr="kontemplate"
;;
blog_cli)
attr="tazjin.blog_cli"
attr="tools.blog_cli"
;;
stern)
attr="stern"
;;
pass)
attr="tazjin.kms_pass"
attr="tools.kms_pass"
;;
*)
echo "The tool '${TARGET_TOOL}' is currently not installed in this repository."

View file

@ -1,8 +1,10 @@
{ buildGoPackage }:
{ pkgs, ... }:
buildGoPackage {
pkgs.buildGoPackage {
name = "blog_cli";
goPackagePath = "github.com/tazjin/personal/blog_cli";
src = ./.;
goDeps = ./deps.nix;
meta.enableCI = true;
}

View file

@ -6,10 +6,10 @@
#
# Only the 'show' and 'insert' commands are supported.
{ google-cloud-sdk, tree, writeShellScriptBin
, project, region, keyring, key }:
{ pkgs, kms, ... }:
writeShellScriptBin "pass" ''
let inherit (pkgs) google-cloud-sdk tree writeShellScriptBin;
in (writeShellScriptBin "pass" ''
set -eo pipefail
CMD="$1"
@ -34,20 +34,20 @@ writeShellScriptBin "pass" ''
show)
secret_check
${google-cloud-sdk}/bin/gcloud kms decrypt \
--project ${project} \
--location ${region} \
--keyring ${keyring} \
--key ${key} \
--project ${kms.project} \
--location ${kms.region} \
--keyring ${kms.keyring} \
--key ${kms.key} \
--ciphertext-file $SECRET_PATH \
--plaintext-file -
;;
insert)
secret_check
${google-cloud-sdk}/bin/gcloud kms encrypt \
--project ${project} \
--location ${region} \
--keyring ${keyring} \
--key ${key} \
--project ${kms.project} \
--location ${kms.region} \
--keyring ${kms.keyring} \
--key ${kms.key} \
--ciphertext-file $SECRET_PATH \
--plaintext-file -
echo "Inserted secret '$SECRET'"
@ -57,4 +57,4 @@ writeShellScriptBin "pass" ''
exit 1
;;
esac
''
'') // { meta.enableCI = true; }