tvl-depot/nix/readTree/default.nix
Aspen Smith 1481066cb7 fix(nix/readTree): Fix typo in docs
Change-Id: I65d6d1d99813d849a02369a0693893ced4681e7e
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10169
Autosubmit: grfn <grfn@gws.fyi>
Tested-by: BuildkiteCI
Reviewed-by: sterni <sternenseemann@systemli.org>
2023-11-30 14:40:44 +00:00

326 lines
10 KiB
Nix
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# Copyright (c) 2019 Vincent Ambo
# Copyright (c) 2020-2021 The TVL Authors
# SPDX-License-Identifier: MIT
#
# Provides a function to automatically read a filesystem structure
# into a Nix attribute set.
#
# Called with an attribute set taking the following arguments:
#
# path: Path to a directory from which to start reading the tree.
#
# args: Argument set to pass to each imported file.
#
# filter: Function to filter `args` based on the tree location. This should
# be a function of the form `args -> location -> args`, where the
# location is a list of strings representing the path components of
# the current readTree target. Optional.
{ ... }:
let
inherit (builtins)
attrNames
concatMap
concatStringsSep
elem
elemAt
filter
hasAttr
head
isAttrs
listToAttrs
map
match
readDir
substring;
argsWithPath = args: parts:
let meta.locatedAt = parts;
in meta // (if isAttrs args then args else args meta);
readDirVisible = path:
let
children = readDir path;
# skip hidden files, except for those that contain special instructions to readTree
isVisible = f: f == ".skip-subtree" || f == ".skip-tree" || (substring 0 1 f) != ".";
names = filter isVisible (attrNames children);
in
listToAttrs (map
(name: {
inherit name;
value = children.${name};
})
names);
# Create a mark containing the location of this attribute and
# a list of all child attribute names added by readTree.
marker = parts: children: {
__readTree = parts;
__readTreeChildren = builtins.attrNames children;
};
# Create a label from a target's tree location.
mkLabel = target:
let label = concatStringsSep "/" target.__readTree;
in if target ? __subtarget
then "${label}:${target.__subtarget}"
else label;
# Merge two attribute sets, but place attributes in `passthru` via
# `overrideAttrs` for derivation targets that support it.
merge = a: b:
if a ? overrideAttrs
then
a.overrideAttrs
(prev: {
passthru = (prev.passthru or { }) // b;
})
else a // b;
# Import a file and enforce our calling convention
importFile = args: scopedArgs: path: parts: filter:
let
importedFile =
if scopedArgs != { } && builtins ? scopedImport # For tvix
then builtins.scopedImport scopedArgs path
else import path;
pathType = builtins.typeOf importedFile;
in
if pathType != "lambda"
then throw "readTree: trying to import ${toString path}, but its a ${pathType}, you need to make it a function like { depot, pkgs, ... }"
else importedFile (filter parts (argsWithPath args parts));
nixFileName = file:
let res = match "(.*)\\.nix" file;
in if res == null then null else head res;
# Internal implementation of readTree, which handles things like the
# skipping of trees and subtrees.
#
# This method returns an attribute sets with either of two shapes:
#
# { ok = ...; } # a tree was read successfully
# { skip = true; } # a tree was skipped
#
# The higher-level `readTree` method assembles the final attribute
# set out of these results at the top-level, and the internal
# `children` implementation unwraps and processes nested trees.
readTreeImpl = { args, initPath, rootDir, parts, argsFilter, scopedArgs }:
let
dir = readDirVisible initPath;
# Determine whether any part of this tree should be skipped.
#
# Adding a `.skip-subtree` file will still allow the import of
# the current node's "default.nix" file, but stop recursion
# there.
#
# Adding a `.skip-tree` file will completely ignore the folder
# in which this file is located.
skipTree = hasAttr ".skip-tree" dir;
skipSubtree = skipTree || hasAttr ".skip-subtree" dir;
joinChild = c: initPath + ("/" + c);
self =
if rootDir
then { __readTree = [ ]; }
else importFile args scopedArgs initPath parts argsFilter;
# Import subdirectories of the current one, unless any skip
# instructions exist.
#
# This file can optionally contain information on why the tree
# should be ignored, but its content is not inspected by
# readTree
filterDir = f: dir."${f}" == "directory";
filteredChildren = map
(c: {
name = c;
value = readTreeImpl {
inherit argsFilter scopedArgs;
args = args;
initPath = (joinChild c);
rootDir = false;
parts = (parts ++ [ c ]);
};
})
(filter filterDir (attrNames dir));
# Remove skipped children from the final set, and unwrap the
# result set.
children =
if skipSubtree then [ ]
else map ({ name, value }: { inherit name; value = value.ok; }) (filter (child: child.value ? ok) filteredChildren);
# Import Nix files
nixFiles =
if skipSubtree then [ ]
else filter (f: f != null) (map nixFileName (attrNames dir));
nixChildren = map
(c:
let
p = joinChild (c + ".nix");
childParts = parts ++ [ c ];
imported = importFile args scopedArgs p childParts argsFilter;
in
{
name = c;
value =
if isAttrs imported
then merge imported (marker childParts { })
else imported;
})
nixFiles;
nodeValue = if dir ? "default.nix" then self else { };
allChildren = listToAttrs (
if dir ? "default.nix"
then children
else nixChildren ++ children
);
in
if skipTree
then { skip = true; }
else {
ok =
if isAttrs nodeValue
then merge nodeValue (allChildren // (marker parts allChildren))
else nodeValue;
};
# Top-level implementation of readTree itself.
readTree = args:
let
tree = readTreeImpl args;
in
if tree ? skip
then throw "Top-level folder has a .skip-tree marker and could not be read by readTree!"
else tree.ok;
# Helper function to fetch subtargets from a target. This is a
# temporary helper to warn on the use of the `meta.targets`
# attribute, which is deprecated in favour of `meta.ci.targets`.
subtargets = node:
let targets = (node.meta.targets or [ ]) ++ (node.meta.ci.targets or [ ]);
in if node ? meta.targets then
builtins.trace ''
Warning: The meta.targets attribute is deprecated.
Please move the subtargets of //${mkLabel node} to the
meta.ci.targets attribute.

''
targets else targets;
# Function which can be used to find all readTree targets within an
# attribute set.
#
# This function will gather physical targets, that is targets which
# correspond directly to a location in the repository, as well as
# subtargets (specified in the meta.ci.targets attribute of a node).
#
# This can be used to discover targets for inclusion in CI
# pipelines.
#
# Called with the arguments:
#
# eligible: Function to determine whether the given derivation
# should be included in the build.
gather = eligible: node:
if node ? __readTree then
# Include the node itself if it is eligible.
(if eligible node then [ node ] else [ ])
# Include eligible children of the node
++ concatMap (gather eligible) (map (attr: node."${attr}") node.__readTreeChildren)
# Include specified sub-targets of the node
++ filter eligible (map
(k: (node."${k}" or { }) // {
# Keep the same tree location, but explicitly mark this
# node as a subtarget.
__readTree = node.__readTree;
__readTreeChildren = [ ];
__subtarget = k;
})
(subtargets node))
else [ ];
# Determine whether a given value is a derivation.
# Copied from nixpkgs/lib for cases where lib is not available yet.
isDerivation = x: isAttrs x && x ? type && x.type == "derivation";
in
{
inherit gather mkLabel;
__functor = _:
{ path
, args
, filter ? (_parts: x: x)
, scopedArgs ? { }
}:
readTree {
inherit args scopedArgs;
argsFilter = filter;
initPath = path;
rootDir = true;
parts = [ ];
};
# In addition to readTree itself, some functionality is exposed that
# is useful for users of readTree.
# Create a readTree filter disallowing access to the specified
# top-level folder in the repository, except for specific exceptions
# specified by their (full) paths.
#
# Called with the arguments:
#
# folder: Name of the restricted top-level folder (e.g. 'experimental')
#
# exceptions: List of readTree parts (e.g. [ [ "services" "some-app" ] ]),
# which should be able to access the restricted folder.
#
# reason: Textual explanation for the restriction (included in errors)
restrictFolder = { folder, exceptions ? [ ], reason }: parts: args:
if (elemAt parts 0) == folder || elem parts exceptions
then args
else args // {
depot = args.depot // {
"${folder}" = throw ''
Access to targets under //${folder} is not permitted from
other repository paths. Specific exceptions are configured
at the top-level.
${reason}
At location: ${builtins.concatStringsSep "." parts}
'';
};
};
# This definition of fix is identical to <nixpkgs>.lib.fix, but is
# provided here for cases where readTree is used before nixpkgs can
# be imported.
#
# It is often required to create the args attribute set.
fix = f: let x = f x; in x;
# Takes an attribute set and adds a meta.ci.targets attribute to it
# which contains all direct children of the attribute set which are
# derivations.
#
# Type: attrs -> attrs
drvTargets = attrs:
attrs // {
# preserve .meta from original attrs
meta = (attrs.meta or { }) // {
# preserve .meta.ci (except .targets) from original attrs
ci = (attrs.meta.ci or { }) // {
targets = builtins.filter
(x: isDerivation attrs."${x}")
(builtins.attrNames attrs);
};
};
};
}