tvl-depot/nix/readTree/default.nix
Griffin Smith b0547ccfa5 feat(nix/readTree): Handle a builtins w/o scopedImport
We (might) not want to implement scopedImport in tvix given it's
considered a bit of a misfeature; this makes readTree work with a
`builtins` set that doesn't have it (and if we decide we do want tvix to
have scopedImport, we can revert this pretty easily).

Change-Id: Ia3bbc847514672063a607d977ce167d489fa1131
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6915
Reviewed-by: sterni <sternenseemann@systemli.org>
Reviewed-by: tazjin <tazjin@tvl.su>
Autosubmit: grfn <grfn@gws.fyi>
Tested-by: BuildkiteCI
2022-10-10 17:42:48 +00:00

284 lines
8.7 KiB
Nix
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# Copyright (c) 2019 Vincent Ambo
# Copyright (c) 2020-2021 The TVL Authors
# SPDX-License-Identifier: MIT
#
# Provides a function to automatically read a a filesystem structure
# into a Nix attribute set.
#
# Called with an attribute set taking the following arguments:
#
# path: Path to a directory from which to start reading the tree.
#
# args: Argument set to pass to each imported file.
#
# filter: Function to filter `args` based on the tree location. This should
# be a function of the form `args -> location -> args`, where the
# location is a list of strings representing the path components of
# the current readTree target. Optional.
{ ... }:
let
inherit (builtins)
attrNames
concatMap
concatStringsSep
elem
elemAt
filter
hasAttr
head
isAttrs
listToAttrs
map
match
readDir
substring;
argsWithPath = args: parts:
let meta.locatedAt = parts;
in meta // (if isAttrs args then args else args meta);
readDirVisible = path:
let
children = readDir path;
isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != ".";
names = filter isVisible (attrNames children);
in
listToAttrs (map
(name: {
inherit name;
value = children.${name};
})
names);
# Create a mark containing the location of this attribute and
# a list of all child attribute names added by readTree.
marker = parts: children: {
__readTree = parts;
__readTreeChildren = builtins.attrNames children;
};
# Create a label from a target's tree location.
mkLabel = target:
let label = concatStringsSep "/" target.__readTree;
in if target ? __subtarget
then "${label}:${target.__subtarget}"
else label;
# Merge two attribute sets, but place attributes in `passthru` via
# `overrideAttrs` for derivation targets that support it.
merge = a: b:
if a ? overrideAttrs
then
a.overrideAttrs
(prev: {
passthru = (prev.passthru or { }) // b;
})
else a // b;
# Import a file and enforce our calling convention
importFile = args: scopedArgs: path: parts: filter:
let
importedFile =
if scopedArgs != { } && builtins ? scopedImport # For tvix
then builtins.scopedImport scopedArgs path
else import path;
pathType = builtins.typeOf importedFile;
in
if pathType != "lambda"
then builtins.throw "readTree: trying to import ${toString path}, but its a ${pathType}, you need to make it a function like { depot, pkgs, ... }"
else importedFile (filter parts (argsWithPath args parts));
nixFileName = file:
let res = match "(.*)\\.nix" file;
in if res == null then null else head res;
readTree = { args, initPath, rootDir, parts, argsFilter, scopedArgs }:
let
dir = readDirVisible initPath;
joinChild = c: initPath + ("/" + c);
self =
if rootDir
then { __readTree = [ ]; }
else importFile args scopedArgs initPath parts argsFilter;
# Import subdirectories of the current one, unless the special
# `.skip-subtree` file exists which makes readTree ignore the
# children.
#
# This file can optionally contain information on why the tree
# should be ignored, but its content is not inspected by
# readTree
filterDir = f: dir."${f}" == "directory";
children = if hasAttr ".skip-subtree" dir then [ ] else
map
(c: {
name = c;
value = readTree {
inherit argsFilter scopedArgs;
args = args;
initPath = (joinChild c);
rootDir = false;
parts = (parts ++ [ c ]);
};
})
(filter filterDir (attrNames dir));
# Import Nix files
nixFiles =
if hasAttr ".skip-subtree" dir then [ ]
else filter (f: f != null) (map nixFileName (attrNames dir));
nixChildren = map
(c:
let
p = joinChild (c + ".nix");
childParts = parts ++ [ c ];
imported = importFile args scopedArgs p childParts argsFilter;
in
{
name = c;
value =
if isAttrs imported
then merge imported (marker childParts { })
else imported;
})
nixFiles;
nodeValue = if dir ? "default.nix" then self else { };
allChildren = listToAttrs (
if dir ? "default.nix"
then children
else nixChildren ++ children
);
in
if isAttrs nodeValue
then merge nodeValue (allChildren // (marker parts allChildren))
else nodeValue;
# Helper function to fetch subtargets from a target. This is a
# temporary helper to warn on the use of the `meta.targets`
# attribute, which is deprecated in favour of `meta.ci.targets`.
subtargets = node:
let targets = (node.meta.targets or [ ]) ++ (node.meta.ci.targets or [ ]);
in if node ? meta.targets then
builtins.trace ''
Warning: The meta.targets attribute is deprecated.
Please move the subtargets of //${mkLabel node} to the
meta.ci.targets attribute.

''
targets else targets;
# Function which can be used to find all readTree targets within an
# attribute set.
#
# This function will gather physical targets, that is targets which
# correspond directly to a location in the repository, as well as
# subtargets (specified in the meta.ci.targets attribute of a node).
#
# This can be used to discover targets for inclusion in CI
# pipelines.
#
# Called with the arguments:
#
# eligible: Function to determine whether the given derivation
# should be included in the build.
gather = eligible: node:
if node ? __readTree then
# Include the node itself if it is eligible.
(if eligible node then [ node ] else [ ])
# Include eligible children of the node
++ concatMap (gather eligible) (map (attr: node."${attr}") node.__readTreeChildren)
# Include specified sub-targets of the node
++ filter eligible (map
(k: (node."${k}" or { }) // {
# Keep the same tree location, but explicitly mark this
# node as a subtarget.
__readTree = node.__readTree;
__readTreeChildren = [ ];
__subtarget = k;
})
(subtargets node))
else [ ];
# Determine whether a given value is a derivation.
# Copied from nixpkgs/lib for cases where lib is not available yet.
isDerivation = x: isAttrs x && x ? type && x.type == "derivation";
in
{
inherit gather mkLabel;
__functor = _:
{ path
, args
, filter ? (_parts: x: x)
, scopedArgs ? { }
}:
readTree {
inherit args scopedArgs;
argsFilter = filter;
initPath = path;
rootDir = true;
parts = [ ];
};
# In addition to readTree itself, some functionality is exposed that
# is useful for users of readTree.
# Create a readTree filter disallowing access to the specified
# top-level folder in the repository, except for specific exceptions
# specified by their (full) paths.
#
# Called with the arguments:
#
# folder: Name of the restricted top-level folder (e.g. 'experimental')
#
# exceptions: List of readTree parts (e.g. [ [ "services" "some-app" ] ]),
# which should be able to access the restricted folder.
#
# reason: Textual explanation for the restriction (included in errors)
restrictFolder = { folder, exceptions ? [ ], reason }: parts: args:
if (elemAt parts 0) == folder || elem parts exceptions
then args
else args // {
depot = args.depot // {
"${folder}" = throw ''
Access to targets under //${folder} is not permitted from
other repository paths. Specific exceptions are configured
at the top-level.
${reason}
At location: ${builtins.concatStringsSep "." parts}
'';
};
};
# This definition of fix is identical to <nixpkgs>.lib.fix, but is
# provided here for cases where readTree is used before nixpkgs can
# be imported.
#
# It is often required to create the args attribute set.
fix = f: let x = f x; in x;
# Takes an attribute set and adds a meta.ci.targets attribute to it
# which contains all direct children of the attribute set which are
# derivations.
#
# Type: attrs -> attrs
drvTargets = attrs:
attrs // {
# preserve .meta from original attrs
meta = (attrs.meta or { }) // {
# preserve .meta.ci (except .targets) from original attrs
ci = (attrs.meta.ci or { }) // {
targets = builtins.filter
(x: isDerivation attrs."${x}")
(builtins.attrNames attrs);
};
};
};
}