tvl-depot/nix/readTree/default.nix

327 lines
10 KiB
Nix
Raw Normal View History

# Copyright (c) 2019 Vincent Ambo
# Copyright (c) 2020-2021 The TVL Authors
# SPDX-License-Identifier: MIT
#
# Provides a function to automatically read a filesystem structure
# into a Nix attribute set.
#
# Called with an attribute set taking the following arguments:
#
# path: Path to a directory from which to start reading the tree.
#
# args: Argument set to pass to each imported file.
#
# filter: Function to filter `args` based on the tree location. This should
# be a function of the form `args -> location -> args`, where the
# location is a list of strings representing the path components of
# the current readTree target. Optional.
{ ... }:
let
inherit (builtins)
attrNames
concatMap
concatStringsSep
elem
elemAt
filter
hasAttr
head
isAttrs
listToAttrs
map
match
readDir
substring;
argsWithPath = args: parts:
let meta.locatedAt = parts;
in meta // (if isAttrs args then args else args meta);
readDirVisible = path:
let
children = readDir path;
# skip hidden files, except for those that contain special instructions to readTree
isVisible = f: f == ".skip-subtree" || f == ".skip-tree" || (substring 0 1 f) != ".";
names = filter isVisible (attrNames children);
in
listToAttrs (map
(name: {
inherit name;
value = children.${name};
})
names);
# Create a mark containing the location of this attribute and
# a list of all child attribute names added by readTree.
marker = parts: children: {
__readTree = parts;
__readTreeChildren = builtins.attrNames children;
};
# Create a label from a target's tree location.
mkLabel = target:
let label = concatStringsSep "/" target.__readTree;
in if target ? __subtarget
then "${label}:${target.__subtarget}"
else label;
# Merge two attribute sets, but place attributes in `passthru` via
# `overrideAttrs` for derivation targets that support it.
merge = a: b:
if a ? overrideAttrs
then
a.overrideAttrs
(prev: {
passthru = (prev.passthru or { }) // b;
})
else a // b;
# Import a file and enforce our calling convention
importFile = args: scopedArgs: path: parts: filter:
let
importedFile =
if scopedArgs != { } && builtins ? scopedImport # For tvix
then builtins.scopedImport scopedArgs path
else import path;
pathType = builtins.typeOf importedFile;
in
if pathType != "lambda"
then throw "readTree: trying to import ${toString path}, but its a ${pathType}, you need to make it a function like { depot, pkgs, ... }"
else importedFile (filter parts (argsWithPath args parts));
nixFileName = file:
let res = match "(.*)\\.nix" file;
in if res == null then null else head res;
# Internal implementation of readTree, which handles things like the
# skipping of trees and subtrees.
#
# This method returns an attribute sets with either of two shapes:
#
# { ok = ...; } # a tree was read successfully
# { skip = true; } # a tree was skipped
#
# The higher-level `readTree` method assembles the final attribute
# set out of these results at the top-level, and the internal
# `children` implementation unwraps and processes nested trees.
readTreeImpl = { args, initPath, rootDir, parts, argsFilter, scopedArgs }:
let
dir = readDirVisible initPath;
# Determine whether any part of this tree should be skipped.
#
# Adding a `.skip-subtree` file will still allow the import of
# the current node's "default.nix" file, but stop recursion
# there.
#
# Adding a `.skip-tree` file will completely ignore the folder
# in which this file is located.
skipTree = hasAttr ".skip-tree" dir;
skipSubtree = skipTree || hasAttr ".skip-subtree" dir;
joinChild = c: initPath + ("/" + c);
self =
if rootDir
then { __readTree = [ ]; }
else importFile args scopedArgs initPath parts argsFilter;
# Import subdirectories of the current one, unless any skip
# instructions exist.
#
# This file can optionally contain information on why the tree
# should be ignored, but its content is not inspected by
# readTree
filterDir = f: dir."${f}" == "directory";
filteredChildren = map
(c: {
name = c;
value = readTreeImpl {
inherit argsFilter scopedArgs;
args = args;
initPath = (joinChild c);
rootDir = false;
parts = (parts ++ [ c ]);
};
})
(filter filterDir (attrNames dir));
# Remove skipped children from the final set, and unwrap the
# result set.
children =
if skipSubtree then [ ]
else map ({ name, value }: { inherit name; value = value.ok; }) (filter (child: child.value ? ok) filteredChildren);
# Import Nix files
nixFiles =
if skipSubtree then [ ]
else filter (f: f != null) (map nixFileName (attrNames dir));
nixChildren = map
(c:
let
p = joinChild (c + ".nix");
childParts = parts ++ [ c ];
imported = importFile args scopedArgs p childParts argsFilter;
in
{
name = c;
value =
if isAttrs imported
then merge imported (marker childParts { })
else imported;
})
nixFiles;
nodeValue = if dir ? "default.nix" then self else { };
allChildren = listToAttrs (
if dir ? "default.nix"
then children
else nixChildren ++ children
);
in
if skipTree
then { skip = true; }
else {
ok =
if isAttrs nodeValue
then merge nodeValue (allChildren // (marker parts allChildren))
else nodeValue;
};
# Top-level implementation of readTree itself.
readTree = args:
let
tree = readTreeImpl args;
in
if tree ? skip
then throw "Top-level folder has a .skip-tree marker and could not be read by readTree!"
else tree.ok;
# Helper function to fetch subtargets from a target. This is a
# temporary helper to warn on the use of the `meta.targets`
# attribute, which is deprecated in favour of `meta.ci.targets`.
subtargets = node:
let targets = (node.meta.targets or [ ]) ++ (node.meta.ci.targets or [ ]);
in if node ? meta.targets then
builtins.trace ''
Warning: The meta.targets attribute is deprecated.
Please move the subtargets of //${mkLabel node} to the
meta.ci.targets attribute.

''
targets else targets;
# Function which can be used to find all readTree targets within an
# attribute set.
#
# This function will gather physical targets, that is targets which
# correspond directly to a location in the repository, as well as
# subtargets (specified in the meta.ci.targets attribute of a node).
#
# This can be used to discover targets for inclusion in CI
# pipelines.
#
# Called with the arguments:
#
# eligible: Function to determine whether the given derivation
# should be included in the build.
gather = eligible: node:
if node ? __readTree then
# Include the node itself if it is eligible.
(if eligible node then [ node ] else [ ])
# Include eligible children of the node
++ concatMap (gather eligible) (map (attr: node."${attr}") node.__readTreeChildren)
# Include specified sub-targets of the node
++ filter eligible (map
(k: (node."${k}" or { }) // {
# Keep the same tree location, but explicitly mark this
# node as a subtarget.
__readTree = node.__readTree;
__readTreeChildren = [ ];
__subtarget = k;
})
(subtargets node))
else [ ];
# Determine whether a given value is a derivation.
# Copied from nixpkgs/lib for cases where lib is not available yet.
isDerivation = x: isAttrs x && x ? type && x.type == "derivation";
in
{
inherit gather mkLabel;
__functor = _:
{ path
, args
, filter ? (_parts: x: x)
, scopedArgs ? { }
}:
readTree {
inherit args scopedArgs;
argsFilter = filter;
initPath = path;
rootDir = true;
parts = [ ];
};
# In addition to readTree itself, some functionality is exposed that
# is useful for users of readTree.
# Create a readTree filter disallowing access to the specified
# top-level folder in the repository, except for specific exceptions
# specified by their (full) paths.
#
# Called with the arguments:
#
# folder: Name of the restricted top-level folder (e.g. 'experimental')
#
# exceptions: List of readTree parts (e.g. [ [ "services" "some-app" ] ]),
# which should be able to access the restricted folder.
#
# reason: Textual explanation for the restriction (included in errors)
restrictFolder = { folder, exceptions ? [ ], reason }: parts: args:
if (elemAt parts 0) == folder || elem parts exceptions
then args
else args // {
depot = args.depot // {
"${folder}" = throw ''
Access to targets under //${folder} is not permitted from
other repository paths. Specific exceptions are configured
at the top-level.
${reason}
At location: ${builtins.concatStringsSep "." parts}
'';
};
};
# This definition of fix is identical to <nixpkgs>.lib.fix, but is
# provided here for cases where readTree is used before nixpkgs can
# be imported.
#
# It is often required to create the args attribute set.
fix = f: let x = f x; in x;
# Takes an attribute set and adds a meta.ci.targets attribute to it
# which contains all direct children of the attribute set which are
# derivations.
#
# Type: attrs -> attrs
drvTargets = attrs:
attrs // {
# preserve .meta from original attrs
meta = (attrs.meta or { }) // {
# preserve .meta.ci (except .targets) from original attrs
ci = (attrs.meta.ci or { }) // {
targets = builtins.filter
(x: isDerivation attrs."${x}")
(builtins.attrNames attrs);
};
};
};
}