chore(tools): remove depot-scanner & tvlc
These are both unused things from a long time ago, which we don't need to keep around anymore. Their design doc has been marked as archived. Change-Id: Icd2744e511e78ec95ec8f39e5f79ed1fe98e9e4a Reviewed-on: https://cl.tvl.fyi/c/depot/+/7639 Autosubmit: tazjin <tazjin@tvl.su> Tested-by: BuildkiteCI Reviewed-by: flokli <flokli@flokli.de>
This commit is contained in:
parent
1e439fb5ad
commit
c6cb138565
10 changed files with 6 additions and 489 deletions
|
@ -1,3 +1,9 @@
|
||||||
|
NOTE: This proposal is archived. We run `josh` instead, and long-term
|
||||||
|
might want to integrate per-target dependency analysis with josh's
|
||||||
|
workspace functionality.
|
||||||
|
|
||||||
|
-------------
|
||||||
|
|
||||||
Below is a prototype for a script to create Git sparse checkouts of the depot.
|
Below is a prototype for a script to create Git sparse checkouts of the depot.
|
||||||
The script below works today with relatively recent versions of git.
|
The script below works today with relatively recent versions of git.
|
||||||
|
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
riking
|
|
|
@ -1,18 +0,0 @@
|
||||||
{ depot, pkgs, ... }:
|
|
||||||
|
|
||||||
let
|
|
||||||
localProto = depot.nix.buildGo.grpc {
|
|
||||||
name = "code.tvl.fyi/tools/depot-scanner/proto";
|
|
||||||
proto = ./depot_scanner.proto;
|
|
||||||
};
|
|
||||||
in
|
|
||||||
depot.nix.buildGo.program
|
|
||||||
{
|
|
||||||
name = "depot-scanner";
|
|
||||||
srcs = [
|
|
||||||
./main.go
|
|
||||||
];
|
|
||||||
deps = [
|
|
||||||
localProto
|
|
||||||
];
|
|
||||||
} // { inherit localProto; }
|
|
|
@ -1,52 +0,0 @@
|
||||||
// Copyright 2020 TVL
|
|
||||||
// SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
syntax = "proto3";
|
|
||||||
package tvl.tools.depot_scanner;
|
|
||||||
option go_package = "code.tvl.fyi/tools/depot-scanner/proto";
|
|
||||||
|
|
||||||
enum PathType {
|
|
||||||
UNKNOWN = 0;
|
|
||||||
DEPOT = 1;
|
|
||||||
STORE = 2;
|
|
||||||
CORE = 3;
|
|
||||||
}
|
|
||||||
|
|
||||||
message ScanRequest {
|
|
||||||
// Which revision of the depot
|
|
||||||
string revision = 1;
|
|
||||||
string attr = 2;
|
|
||||||
// Optionally, the attr to evaluate can be provided as a path to a folder or a
|
|
||||||
// .nix file. This is used by the HTTP service.
|
|
||||||
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
|
|
||||||
string attrAsPath = 3;
|
|
||||||
}
|
|
||||||
|
|
||||||
message ScanResponse {
|
|
||||||
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
|
|
||||||
repeated string depotPath = 1;
|
|
||||||
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
|
|
||||||
repeated string nixStorePath = 2;
|
|
||||||
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
|
|
||||||
repeated string corePkgsPath = 4;
|
|
||||||
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
|
|
||||||
repeated string otherPath = 3;
|
|
||||||
|
|
||||||
bytes derivation = 5;
|
|
||||||
}
|
|
||||||
|
|
||||||
message ArchiveRequest {
|
|
||||||
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
|
|
||||||
repeated string depotPath = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
message ArchiveChunk {
|
|
||||||
bytes chunk = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
service DepotScanService {
|
|
||||||
rpc Scan(ScanRequest) returns (ScanResponse);
|
|
||||||
|
|
||||||
rpc MakeArchive(ArchiveRequest) returns (stream ArchiveChunk);
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,3 +0,0 @@
|
||||||
module code.tvl.fyi/tools/depot-scanner
|
|
||||||
|
|
||||||
go 1.14
|
|
|
@ -1,227 +0,0 @@
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
pb "code.tvl.fyi/tools/depot-scanner/proto"
|
|
||||||
)
|
|
||||||
|
|
||||||
var nixInstantiatePath = flag.String("nix-bin", "/run/current-system/sw/bin/nix-instantiate", "path to nix-instantiate")
|
|
||||||
var depotRoot = flag.String("depot", envOr("DEPOT_ROOT", "/depot/"), "path to tvl.fyi depot at current canon")
|
|
||||||
var nixStoreRoot = flag.String("store-path", "/nix/store/", "prefix for all valid nix store paths")
|
|
||||||
|
|
||||||
var modeFlag = flag.String("mode", modeArchive, "operation mode. valid values: tar, print")
|
|
||||||
var onlyFlag = flag.String("only", "", "only enable the listed output types, comma separated. valid values: DEPOT, STORE, CORE, UNKNOWN")
|
|
||||||
var relativeFlag = flag.Bool("relpath", false, "when printing paths, print them relative to the root of their path type")
|
|
||||||
|
|
||||||
const (
|
|
||||||
modeArchive = "tar"
|
|
||||||
modePrint = "print"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// String that identifies a path as belonging to nix corepkgs.
|
|
||||||
corePkgsString = "/share/nix/corepkgs/"
|
|
||||||
|
|
||||||
depotTraceString = "trace: depot-scan: "
|
|
||||||
)
|
|
||||||
|
|
||||||
type fileScanType int
|
|
||||||
|
|
||||||
const (
|
|
||||||
unknownPath fileScanType = iota
|
|
||||||
depotPath
|
|
||||||
nixStorePath
|
|
||||||
corePkgsPath
|
|
||||||
)
|
|
||||||
|
|
||||||
func launchNix(attr string) (*exec.Cmd, io.ReadCloser, io.ReadCloser, error) {
|
|
||||||
cmd := exec.Command(*nixInstantiatePath, "--trace-file-access", "-A", attr)
|
|
||||||
stdout, err := cmd.StdoutPipe()
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
stderr, err := cmd.StderrPipe()
|
|
||||||
if err != nil {
|
|
||||||
stdout.Close()
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = cmd.Start()
|
|
||||||
if err != nil {
|
|
||||||
stdout.Close()
|
|
||||||
stderr.Close()
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return cmd, stdout, stderr, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func categorizePath(path string) fileScanType {
|
|
||||||
if strings.HasPrefix(path, *nixStoreRoot) {
|
|
||||||
if strings.Contains(path, corePkgsString) {
|
|
||||||
return corePkgsPath
|
|
||||||
}
|
|
||||||
return nixStorePath
|
|
||||||
} else if strings.HasPrefix(path, *depotRoot) {
|
|
||||||
return depotPath
|
|
||||||
} else if strings.Contains(path, corePkgsString) {
|
|
||||||
return corePkgsPath
|
|
||||||
}
|
|
||||||
return unknownPath
|
|
||||||
}
|
|
||||||
|
|
||||||
func addPath(path string, out map[fileScanType]map[string]struct{}) {
|
|
||||||
cat := categorizePath(path)
|
|
||||||
if out[cat] == nil {
|
|
||||||
out[cat] = make(map[string]struct{})
|
|
||||||
}
|
|
||||||
|
|
||||||
out[cat][path] = struct{}{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func consumeOutput(stdout, stderr io.ReadCloser) (map[fileScanType]map[string]struct{}, string, error) {
|
|
||||||
result := make(map[fileScanType]map[string]struct{})
|
|
||||||
|
|
||||||
scanner := bufio.NewScanner(stderr)
|
|
||||||
for scanner.Scan() {
|
|
||||||
line := scanner.Text()
|
|
||||||
if strings.HasPrefix(line, depotTraceString) {
|
|
||||||
addPath(strings.TrimPrefix(line, depotTraceString), result)
|
|
||||||
} else {
|
|
||||||
// print remaining stderr output of nix-instantiate
|
|
||||||
// to prevent silent swallowing of possible important
|
|
||||||
// error messages (e.g. about command line interface changes)
|
|
||||||
fmt.Fprintf(os.Stderr, "nix-inst> %s\n", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if scanner.Err() != nil {
|
|
||||||
return nil, "", scanner.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get derivation path
|
|
||||||
derivPath := ""
|
|
||||||
scanner = bufio.NewScanner(stdout)
|
|
||||||
for scanner.Scan() {
|
|
||||||
line := scanner.Text()
|
|
||||||
if strings.HasPrefix(line, *nixStoreRoot) {
|
|
||||||
derivPath = line
|
|
||||||
// consume the rest of the output
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if scanner.Err() != nil {
|
|
||||||
return nil, "", scanner.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
return result, derivPath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
checkDepotRoot()
|
|
||||||
|
|
||||||
enabledPathTypes := make(map[pb.PathType]bool, 4)
|
|
||||||
if len(*onlyFlag) > 0 {
|
|
||||||
enabledOutputs := strings.Split(*onlyFlag, ",")
|
|
||||||
for _, v := range enabledOutputs {
|
|
||||||
i, ok := pb.PathType_value[strings.ToUpper(v)]
|
|
||||||
if !ok {
|
|
||||||
fmt.Fprintln(os.Stderr, "warning: unrecognized PathType name: ", v)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
enabledPathTypes[pb.PathType(i)] = true
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Default
|
|
||||||
enabledPathTypes = map[pb.PathType]bool{
|
|
||||||
pb.PathType_UNKNOWN: true,
|
|
||||||
pb.PathType_DEPOT: true,
|
|
||||||
pb.PathType_STORE: true,
|
|
||||||
pb.PathType_CORE: true,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd, stdout, stderr, err := launchNix(flag.Arg(0))
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Errorf("could not launch nix: %w", err))
|
|
||||||
}
|
|
||||||
results, derivPath, err := consumeOutput(stdout, stderr)
|
|
||||||
if err != nil {
|
|
||||||
err2 := cmd.Wait()
|
|
||||||
if err2 != nil {
|
|
||||||
panic(fmt.Errorf("nix-instantiate failed: %w\nadditionally, while reading output: %w", err2, err))
|
|
||||||
}
|
|
||||||
panic(fmt.Errorf("problem reading nix output: %w", err))
|
|
||||||
}
|
|
||||||
err = cmd.Wait()
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Errorf("nix-instantiate failed: %w", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = derivPath
|
|
||||||
|
|
||||||
if *modeFlag == "print" {
|
|
||||||
if enabledPathTypes[pb.PathType_STORE] {
|
|
||||||
for k, _ := range results[nixStorePath] {
|
|
||||||
if *relativeFlag {
|
|
||||||
k = strings.TrimPrefix(k, *nixStoreRoot)
|
|
||||||
k = strings.TrimPrefix(k, "/")
|
|
||||||
}
|
|
||||||
fmt.Println(k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if enabledPathTypes[pb.PathType_DEPOT] {
|
|
||||||
for k, _ := range results[depotPath] {
|
|
||||||
if *relativeFlag {
|
|
||||||
k = strings.TrimPrefix(k, *depotRoot)
|
|
||||||
k = strings.TrimPrefix(k, "/")
|
|
||||||
}
|
|
||||||
fmt.Println(k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if enabledPathTypes[pb.PathType_CORE] {
|
|
||||||
for k, _ := range results[corePkgsPath] {
|
|
||||||
// TODO relativeFlag
|
|
||||||
fmt.Println(k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if enabledPathTypes[pb.PathType_UNKNOWN] {
|
|
||||||
for k, _ := range results[unknownPath] {
|
|
||||||
fmt.Println(k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
panic("unimplemented")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func envOr(envVar, def string) string {
|
|
||||||
v := os.Getenv(envVar)
|
|
||||||
if v == "" {
|
|
||||||
return def
|
|
||||||
}
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkDepotRoot() {
|
|
||||||
if *depotRoot == "" {
|
|
||||||
fmt.Fprintln(os.Stderr, "error: DEPOT_ROOT / -depot not set")
|
|
||||||
os.Exit(2)
|
|
||||||
}
|
|
||||||
_, err := os.Stat(*depotRoot)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
fmt.Fprintf(os.Stderr, "error: %q does not exist\ndid you forget to set DEPOT_ROOT / --depot ?\n", *depotRoot)
|
|
||||||
os.Exit(1)
|
|
||||||
} else if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "error: could not stat %q: %v\n", *depotRoot, err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1 +0,0 @@
|
||||||
riking
|
|
|
@ -1,33 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -eu
|
|
||||||
set -o pipefail
|
|
||||||
|
|
||||||
source path-scripts
|
|
||||||
|
|
||||||
XDG_DATA_HOME="${XDG_DATA_HOME:-$HOME/.local/share}"
|
|
||||||
tvlc_root="$XDG_DATA_HOME/tvlc"
|
|
||||||
|
|
||||||
nice_checkout_root=
|
|
||||||
if [ -f "$tvlc_root"/nice_checkout_root ]; then
|
|
||||||
nice_checkout_root="$(cat "$tvlc_root"/nice_checkout_root)"
|
|
||||||
fi
|
|
||||||
nice_checkout_root="${nice_checkout_root:-$HOME/tvlc}"
|
|
||||||
|
|
||||||
depot_root=
|
|
||||||
if [ -f "$tvlc_root/depot_root" ]; then
|
|
||||||
depot_root="$(cat "$tvlc_root/depot_root")"
|
|
||||||
fi
|
|
||||||
if [ -d /depot ]; then
|
|
||||||
# don't require config on tvl nixos servers
|
|
||||||
depot_root="${depot_root:-/depot}"
|
|
||||||
fi
|
|
||||||
if [ -n "$depot_root" ]; then
|
|
||||||
export DEPOT_ROOT="$depot_root"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ! -d "$tvlc_root" ]; then
|
|
||||||
echo "tvlc: setup required"
|
|
||||||
echo "please run 'tvlc setup' from the depot root"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
|
@ -1,51 +0,0 @@
|
||||||
{ pkgs, depot, ... }:
|
|
||||||
|
|
||||||
let
|
|
||||||
pathScripts = pkgs.writeShellScript "imports" ''
|
|
||||||
export tvix_instantiate="${depot.third_party.nix}/bin/nix-instantiate"
|
|
||||||
export depot_scanner="${depot.tools.depot-scanner}/bin/depot-scanner"
|
|
||||||
'';
|
|
||||||
|
|
||||||
# setup: git rev-parse --show-toplevel > $tvlc_root/depot_root
|
|
||||||
# setup: mkdir $tvlc_root/clients
|
|
||||||
# setup: echo 1 > $tvlc_root/next_clientid
|
|
||||||
|
|
||||||
commonsh = pkgs.stdenv.mkDerivation {
|
|
||||||
name = "common.sh";
|
|
||||||
src = ./common.sh;
|
|
||||||
doCheck = true;
|
|
||||||
unpackPhase = "true";
|
|
||||||
buildPhase = ''
|
|
||||||
substitute ${./common.sh} $out --replace path-scripts ${pathScripts}
|
|
||||||
'';
|
|
||||||
checkPhase = ''
|
|
||||||
${pkgs.shellcheck}/bin/shellcheck $out ${pathScripts} && echo "SHELLCHECK OK"
|
|
||||||
'';
|
|
||||||
installPhase = ''
|
|
||||||
chmod +x $out
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
tvlcNew = pkgs.stdenv.mkDerivation {
|
|
||||||
name = "tvlc-new";
|
|
||||||
src = ./tvlc-new;
|
|
||||||
doCheck = true;
|
|
||||||
|
|
||||||
unpackPhase = "true";
|
|
||||||
buildPhase = ''
|
|
||||||
substitute ${./tvlc-new} $out --replace common.sh ${commonsh}
|
|
||||||
'';
|
|
||||||
checkPhase = ''
|
|
||||||
${pkgs.shellcheck}/bin/shellcheck $out ${commonsh} ${pathScripts} && echo "SHELLCHECK OK"
|
|
||||||
'';
|
|
||||||
installPhase = ''
|
|
||||||
chmod +x $out
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
|
|
||||||
in
|
|
||||||
{
|
|
||||||
inherit pathScripts;
|
|
||||||
inherit commonsh;
|
|
||||||
inherit tvlcNew;
|
|
||||||
}
|
|
|
@ -1,103 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
source common.sh
|
|
||||||
|
|
||||||
set -eu
|
|
||||||
set -o pipefail
|
|
||||||
|
|
||||||
function usage() {
|
|
||||||
echo "tvlc new [-n|--name CLIENTNAME] [derivation...]"
|
|
||||||
echo ""
|
|
||||||
cat <<EOF
|
|
||||||
The 'new' command creates a new git sparse checkout with the given name, and
|
|
||||||
contents needed to build the Nix derivation(s) specified on the command line.
|
|
||||||
|
|
||||||
Options:
|
|
||||||
-n/--name client-name: Sets the git branch and nice checkout name for the
|
|
||||||
workspace. If the option is not provided, the name will be based on the
|
|
||||||
first non-option command-line argument.
|
|
||||||
--branch branch-name: Sets the git branch name only.
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
checkout_name=
|
|
||||||
branch_name=
|
|
||||||
|
|
||||||
options=$(getopt -o 'n:' --long debug --long name: -- "$@")
|
|
||||||
eval set -- "$options"
|
|
||||||
while true; do
|
|
||||||
case "$1" in
|
|
||||||
-h)
|
|
||||||
usage
|
|
||||||
exit 0
|
|
||||||
;;
|
|
||||||
-v)
|
|
||||||
version
|
|
||||||
exit 0
|
|
||||||
;;
|
|
||||||
-n|--name)
|
|
||||||
shift
|
|
||||||
checkout_name="$1"
|
|
||||||
if [ -z "$branch_name" ]; then
|
|
||||||
branch_name=tvlc-"$1"
|
|
||||||
fi
|
|
||||||
;;
|
|
||||||
--branch)
|
|
||||||
shift
|
|
||||||
branch_name="$1"
|
|
||||||
;;
|
|
||||||
--)
|
|
||||||
shift
|
|
||||||
break
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
shift
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ $# -eq 0 ]; then
|
|
||||||
echo "error: workspace name, target derivations required"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "$checkout_name" ]; then
|
|
||||||
# TODO(riking): deduce
|
|
||||||
echo "error: workspace name (-n) required"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -d "$nice_checkout_root/$checkout_name" ]; then
|
|
||||||
echo "error: checkout $checkout_name already exists"
|
|
||||||
# nb: shellescape checkout_name because we expect the user to copy-paste it
|
|
||||||
# shellcheck disable=SC1003
|
|
||||||
echo "consider deleting it with tvlc remove '${checkout_name/'/\'}'"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if [ -f "$DEPOT_ROOT/.git/refs/heads/$branch_name" ]; then
|
|
||||||
echo "error: branch $branch_name already exists in git"
|
|
||||||
# shellcheck disable=SC1003
|
|
||||||
echo "consider deleting it with cd $DEPOT_ROOT; git branch -d '${checkout_name/'/\'}'"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# The big one: call into Nix to figure out what paths the desired derivations depend on.
|
|
||||||
readarray -t includedPaths < <("$depot_scanner" --mode 'print' --only 'DEPOT' --relpath --depot "$DEPOT_ROOT" --nix-bin "$tvix_instantiate" "$@")
|
|
||||||
|
|
||||||
# bash math
|
|
||||||
checkout_id=$(("$(cat "$tvlc_root/next_clientid")"))
|
|
||||||
next_checkout_id=$(("$checkout_id"+1))
|
|
||||||
echo "$next_checkout_id" > "$tvlc_root/next_clientid"
|
|
||||||
|
|
||||||
checkout_dir="$tvlc_root/clients/$checkout_id"
|
|
||||||
mkdir "$checkout_dir"
|
|
||||||
cd "$DEPOT_ROOT"
|
|
||||||
git worktree add --no-checkout -b "$branch_name" "$checkout_dir"
|
|
||||||
# BUG: git not creating the /info/ subdir
|
|
||||||
mkdir "$DEPOT_ROOT/.git/worktrees/$checkout_id/info"
|
|
||||||
|
|
||||||
cd "$checkout_dir"
|
|
||||||
git sparse-checkout init --cone
|
|
||||||
git sparse-checkout set "${includedPaths[@]}"
|
|
||||||
|
|
||||||
ln -s "$checkout_dir" "$nice_checkout_root"/"$checkout_name"
|
|
||||||
|
|
||||||
echo "$nice_checkout_root/$checkout_name"
|
|
Loading…
Reference in a new issue