feat(nix/runExecline): add runExecline

runExecline is a primitive that just does not care.

It’s similar to `runCommand`, but instead of concatenating bash
scripts left and right, it actually *uses* the features of
`derivation`, passing things to `args` and making it possible to
overwrite the `builder` in a sensible manner.

Additionally, it provides a way to pass a nix string to `stdin` of the
build script.

Similar to `writeExecline`, the passed script is not a string, but a
nested list of nix lists representing execline blocks. Escaping is
done by the implementation, the user can just use normal nix strings.

Change-Id: I890d9e5d921207751cdc8cc4309381395d92742f
Reviewed-on: https://cl.tvl.fyi/c/depot/+/701
Reviewed-by: BuildkiteCI
Reviewed-by: isomer <isomer@tvl.fyi>
Reviewed-by: tazjin <mail@tazj.in>
Tested-by: BuildkiteCI
This commit is contained in:
Profpatsch 2020-06-28 03:14:11 +02:00
parent 7a6a007302
commit 4402c363b6
5 changed files with 254 additions and 1 deletions

View file

@ -73,7 +73,9 @@ in lib.fix (self: {
tools.cheddar
tools.nsfv-setup
(drvify "getBins-tests" nix.getBins.tests)
];
]
++ nix.runExecline.tests
;
# Haskell packages we've patched locally
haskellPackages = with depot.third_party.haskellPackages; [

3
nix/runExecline/OWNERS Normal file
View file

@ -0,0 +1,3 @@
inherited: true
owners:
- Profpatsch

View file

@ -0,0 +1,19 @@
{ depot, pkgs, lib, ... }:
let
runExecline = import ./runExecline.nix {
inherit (pkgs) stdenv;
inherit (depot.nix) escapeExecline getBins;
inherit pkgs lib;
};
tests = import ./tests.nix {
inherit runExecline;
inherit (depot.nix) getBins;
inherit (pkgs) stdenv coreutils;
inherit pkgs;
};
in {
__functor = _: runExecline;
inherit tests;
}

View file

@ -0,0 +1,121 @@
{ pkgs, stdenv, lib, getBins, escapeExecline }:
# runExecline is a primitive building block
# for writing non-kitchen sink builders.
#
# Its conceptually similar to `runCommand`,
# but instead of concatenating bash scripts left
# and right, it actually *uses* the features of
# `derivation`, passing things to `args`
# and making it possible to overwrite the `builder`
# in a sensible manner.
#
# Additionally, it provides a way to pass a nix string
# to `stdin` of the build script.
#
# Similar to //nix/writeExecline, the passed script is
# not a string, but a nested list of nix lists
# representing execline blocks. Escaping is
# done by the implementation, the user can just use
# normal nix strings.
#
# Example:
#
# runExecline "my-drv" { stdin = "hi!"; } [
# "importas" "out" "out"
# # this pipes stdout of s6-cat to $out
# # and s6-cat redirects from stdin to stdout
# "redirfd" "-w" "1" "$out" bins.s6-cat
# ]
#
# which creates a derivation with "hi!" in $out.
#
# See ./tests.nix for more examples.
let
bins = getBins pkgs.execline [
"execlineb"
{ use = "if"; as = "execlineIf"; }
"redirfd"
"importas"
"exec"
]
// getBins pkgs.s6-portable-utils [
"s6-cat"
"s6-grep"
"s6-touch"
"s6-test"
"s6-chmod"
];
in
name:
{
# a string to pass as stdin to the execline script
stdin ? ""
# a program wrapping the acutal execline invocation;
# should be in Bernstein-chaining style
, builderWrapper ? bins.exec
# additional arguments to pass to the derivation
, derivationArgs ? {}
}:
# the execline script as a nested list of string,
# representing the blocks;
# see docs of `escapeExecline`.
execline:
# those arguments cant be overwritten
assert !derivationArgs ? system;
assert !derivationArgs ? name;
assert !derivationArgs ? builder;
assert !derivationArgs ? args;
derivation (derivationArgs // {
# TODO(Profpatsch): what about cross?
inherit (stdenv) system;
inherit name;
# okay, `builtins.toFile` does not accept strings
# that reference drv outputs. This means we need
# to pass the script and stdin as envvar;
# this might clash with another passed envar,
# so we give it a long & unique name
_runExeclineScript =
let
in escapeExecline execline;
_runExeclineStdin = stdin;
passAsFile = [
"_runExeclineScript"
"_runExeclineStdin"
] ++ derivationArgs.passAsFile or [];
# the default, exec acts as identity executable
builder = builderWrapper;
args = [
bins.importas # import script file as $script
"-ui" # drop the envvar afterwards
"script" # substitution name
"_runExeclineScriptPath" # passed script file
bins.importas # do the same for $stdin
"-ui"
"stdin"
"_runExeclineStdinPath"
bins.redirfd # now we
"-r" # read the file
"0" # into the stdin of execlineb
"$stdin" # that was given via stdin
bins.execlineb # the actual invocation
# TODO(Profpatsch): depending on the use-case, -S0 might not be enough
# in all use-cases, then a wrapper for execlineb arguments
# should be added (-P, -S, -s).
"-S0" # set $@ inside the execline script
"-W" # die on syntax error
"$script" # substituted by importas
];
})

108
nix/runExecline/tests.nix Normal file
View file

@ -0,0 +1,108 @@
{ stdenv, pkgs, runExecline, getBins
# https://www.mail-archive.com/skaware@list.skarnet.org/msg01256.html
, coreutils }:
let
bins = getBins coreutils [ "mv" ]
// getBins pkgs.execline [
"execlineb"
{ use = "if"; as = "execlineIf"; }
"redirfd"
"importas"
]
// getBins pkgs.s6-portable-utils [
"s6-chmod"
"s6-grep"
"s6-touch"
"s6-cat"
"s6-test"
];
# lol
writeScript = name: script: runExecline name {
derivationArgs = {
inherit script;
passAsFile = [ "script" ];
preferLocalBuild = true;
allowSubstitutes = false;
};
} [
"importas" "-ui" "s" "scriptPath"
"importas" "-ui" "out" "out"
"foreground" [
bins.mv "$s" "$out"
]
bins.s6-chmod "0755" "$out"
];
# execline block of depth 1
block = args: builtins.map (arg: " ${arg}") args ++ [ "" ];
# derivation that tests whether a given line exists
# in the given file. Does not use runExecline, because
# that should be tested after all.
fileHasLine = line: file: derivation {
name = "run-execline-test-file-${file.name}-has-line";
inherit (stdenv) system;
builder = bins.execlineIf;
args =
(block [
bins.redirfd "-r" "0" file # read file to stdin
bins.s6-grep "-F" "-q" line # and grep for the line
])
++ [
# if the block succeeded, touch $out
bins.importas "-ui" "out" "out"
bins.s6-touch "$out"
];
preferLocalBuild = true;
allowSubstitutes = false;
};
# basic test that touches out
basic = runExecline "run-execline-test-basic" {
derivationArgs = {
preferLocalBuild = true;
allowSubstitutes = false;
};
} [
"importas" "-ui" "out" "out"
"${bins.s6-touch}" "$out"
];
# whether the stdin argument works as intended
stdin = fileHasLine "foo" (runExecline "run-execline-test-stdin" {
stdin = "foo\nbar\nfoo";
derivationArgs = {
preferLocalBuild = true;
allowSubstitutes = false;
};
} [
"importas" "-ui" "out" "out"
# this pipes stdout of s6-cat to $out
# and s6-cat redirects from stdin to stdout
"redirfd" "-w" "1" "$out" bins.s6-cat
]);
wrapWithVar = runExecline "run-execline-test-wrap-with-var" {
builderWrapper = writeScript "var-wrapper" ''
#!${bins.execlineb} -S0
export myvar myvalue $@
'';
derivationArgs = {
preferLocalBuild = true;
allowSubstitutes = false;
};
} [
"importas" "-ui" "v" "myvar"
"if" [ bins.s6-test "myvalue" "=" "$v" ]
"importas" "out" "out"
bins.s6-touch "$out"
];
in [
basic
stdin
wrapWithVar
]