chore(tools): remove depot-scanner & tvlc

These are both unused things from a long time ago, which we don't need
to keep around anymore.

Their design doc has been marked as archived.

Change-Id: Icd2744e511e78ec95ec8f39e5f79ed1fe98e9e4a
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7639
Autosubmit: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
This commit is contained in:
Vincent Ambo 2022-12-26 12:49:53 +03:00 committed by tazjin
parent 1e439fb5ad
commit c6cb138565
10 changed files with 6 additions and 489 deletions

View file

@ -1,3 +1,9 @@
NOTE: This proposal is archived. We run `josh` instead, and long-term
might want to integrate per-target dependency analysis with josh's
workspace functionality.
-------------
Below is a prototype for a script to create Git sparse checkouts of the depot. Below is a prototype for a script to create Git sparse checkouts of the depot.
The script below works today with relatively recent versions of git. The script below works today with relatively recent versions of git.

View file

@ -1 +0,0 @@
riking

View file

@ -1,18 +0,0 @@
{ depot, pkgs, ... }:
let
localProto = depot.nix.buildGo.grpc {
name = "code.tvl.fyi/tools/depot-scanner/proto";
proto = ./depot_scanner.proto;
};
in
depot.nix.buildGo.program
{
name = "depot-scanner";
srcs = [
./main.go
];
deps = [
localProto
];
} // { inherit localProto; }

View file

@ -1,52 +0,0 @@
// Copyright 2020 TVL
// SPDX-License-Identifier: MIT
syntax = "proto3";
package tvl.tools.depot_scanner;
option go_package = "code.tvl.fyi/tools/depot-scanner/proto";
enum PathType {
UNKNOWN = 0;
DEPOT = 1;
STORE = 2;
CORE = 3;
}
message ScanRequest {
// Which revision of the depot
string revision = 1;
string attr = 2;
// Optionally, the attr to evaluate can be provided as a path to a folder or a
// .nix file. This is used by the HTTP service.
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
string attrAsPath = 3;
}
message ScanResponse {
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
repeated string depotPath = 1;
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
repeated string nixStorePath = 2;
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
repeated string corePkgsPath = 4;
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
repeated string otherPath = 3;
bytes derivation = 5;
}
message ArchiveRequest {
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
repeated string depotPath = 1;
}
message ArchiveChunk {
bytes chunk = 1;
}
service DepotScanService {
rpc Scan(ScanRequest) returns (ScanResponse);
rpc MakeArchive(ArchiveRequest) returns (stream ArchiveChunk);
}

View file

@ -1,3 +0,0 @@
module code.tvl.fyi/tools/depot-scanner
go 1.14

View file

@ -1,227 +0,0 @@
package main
import (
"bufio"
"flag"
"fmt"
"io"
"os"
"os/exec"
"strings"
pb "code.tvl.fyi/tools/depot-scanner/proto"
)
var nixInstantiatePath = flag.String("nix-bin", "/run/current-system/sw/bin/nix-instantiate", "path to nix-instantiate")
var depotRoot = flag.String("depot", envOr("DEPOT_ROOT", "/depot/"), "path to tvl.fyi depot at current canon")
var nixStoreRoot = flag.String("store-path", "/nix/store/", "prefix for all valid nix store paths")
var modeFlag = flag.String("mode", modeArchive, "operation mode. valid values: tar, print")
var onlyFlag = flag.String("only", "", "only enable the listed output types, comma separated. valid values: DEPOT, STORE, CORE, UNKNOWN")
var relativeFlag = flag.Bool("relpath", false, "when printing paths, print them relative to the root of their path type")
const (
modeArchive = "tar"
modePrint = "print"
)
const (
// String that identifies a path as belonging to nix corepkgs.
corePkgsString = "/share/nix/corepkgs/"
depotTraceString = "trace: depot-scan: "
)
type fileScanType int
const (
unknownPath fileScanType = iota
depotPath
nixStorePath
corePkgsPath
)
func launchNix(attr string) (*exec.Cmd, io.ReadCloser, io.ReadCloser, error) {
cmd := exec.Command(*nixInstantiatePath, "--trace-file-access", "-A", attr)
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, nil, nil, err
}
stderr, err := cmd.StderrPipe()
if err != nil {
stdout.Close()
return nil, nil, nil, err
}
err = cmd.Start()
if err != nil {
stdout.Close()
stderr.Close()
return nil, nil, nil, err
}
return cmd, stdout, stderr, nil
}
func categorizePath(path string) fileScanType {
if strings.HasPrefix(path, *nixStoreRoot) {
if strings.Contains(path, corePkgsString) {
return corePkgsPath
}
return nixStorePath
} else if strings.HasPrefix(path, *depotRoot) {
return depotPath
} else if strings.Contains(path, corePkgsString) {
return corePkgsPath
}
return unknownPath
}
func addPath(path string, out map[fileScanType]map[string]struct{}) {
cat := categorizePath(path)
if out[cat] == nil {
out[cat] = make(map[string]struct{})
}
out[cat][path] = struct{}{}
}
func consumeOutput(stdout, stderr io.ReadCloser) (map[fileScanType]map[string]struct{}, string, error) {
result := make(map[fileScanType]map[string]struct{})
scanner := bufio.NewScanner(stderr)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, depotTraceString) {
addPath(strings.TrimPrefix(line, depotTraceString), result)
} else {
// print remaining stderr output of nix-instantiate
// to prevent silent swallowing of possible important
// error messages (e.g. about command line interface changes)
fmt.Fprintf(os.Stderr, "nix-inst> %s\n", line)
}
}
if scanner.Err() != nil {
return nil, "", scanner.Err()
}
// Get derivation path
derivPath := ""
scanner = bufio.NewScanner(stdout)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, *nixStoreRoot) {
derivPath = line
// consume the rest of the output
}
}
if scanner.Err() != nil {
return nil, "", scanner.Err()
}
return result, derivPath, nil
}
func main() {
flag.Parse()
checkDepotRoot()
enabledPathTypes := make(map[pb.PathType]bool, 4)
if len(*onlyFlag) > 0 {
enabledOutputs := strings.Split(*onlyFlag, ",")
for _, v := range enabledOutputs {
i, ok := pb.PathType_value[strings.ToUpper(v)]
if !ok {
fmt.Fprintln(os.Stderr, "warning: unrecognized PathType name: ", v)
continue
}
enabledPathTypes[pb.PathType(i)] = true
}
} else {
// Default
enabledPathTypes = map[pb.PathType]bool{
pb.PathType_UNKNOWN: true,
pb.PathType_DEPOT: true,
pb.PathType_STORE: true,
pb.PathType_CORE: true,
}
}
cmd, stdout, stderr, err := launchNix(flag.Arg(0))
if err != nil {
panic(fmt.Errorf("could not launch nix: %w", err))
}
results, derivPath, err := consumeOutput(stdout, stderr)
if err != nil {
err2 := cmd.Wait()
if err2 != nil {
panic(fmt.Errorf("nix-instantiate failed: %w\nadditionally, while reading output: %w", err2, err))
}
panic(fmt.Errorf("problem reading nix output: %w", err))
}
err = cmd.Wait()
if err != nil {
panic(fmt.Errorf("nix-instantiate failed: %w", err))
}
_ = derivPath
if *modeFlag == "print" {
if enabledPathTypes[pb.PathType_STORE] {
for k, _ := range results[nixStorePath] {
if *relativeFlag {
k = strings.TrimPrefix(k, *nixStoreRoot)
k = strings.TrimPrefix(k, "/")
}
fmt.Println(k)
}
}
if enabledPathTypes[pb.PathType_DEPOT] {
for k, _ := range results[depotPath] {
if *relativeFlag {
k = strings.TrimPrefix(k, *depotRoot)
k = strings.TrimPrefix(k, "/")
}
fmt.Println(k)
}
}
if enabledPathTypes[pb.PathType_CORE] {
for k, _ := range results[corePkgsPath] {
// TODO relativeFlag
fmt.Println(k)
}
}
if enabledPathTypes[pb.PathType_UNKNOWN] {
for k, _ := range results[unknownPath] {
fmt.Println(k)
}
}
} else {
panic("unimplemented")
}
}
func envOr(envVar, def string) string {
v := os.Getenv(envVar)
if v == "" {
return def
}
return v
}
func checkDepotRoot() {
if *depotRoot == "" {
fmt.Fprintln(os.Stderr, "error: DEPOT_ROOT / -depot not set")
os.Exit(2)
}
_, err := os.Stat(*depotRoot)
if os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "error: %q does not exist\ndid you forget to set DEPOT_ROOT / --depot ?\n", *depotRoot)
os.Exit(1)
} else if err != nil {
fmt.Fprintf(os.Stderr, "error: could not stat %q: %v\n", *depotRoot, err)
os.Exit(1)
}
}

View file

@ -1 +0,0 @@
riking

View file

@ -1,33 +0,0 @@
#!/bin/bash
set -eu
set -o pipefail
source path-scripts
XDG_DATA_HOME="${XDG_DATA_HOME:-$HOME/.local/share}"
tvlc_root="$XDG_DATA_HOME/tvlc"
nice_checkout_root=
if [ -f "$tvlc_root"/nice_checkout_root ]; then
nice_checkout_root="$(cat "$tvlc_root"/nice_checkout_root)"
fi
nice_checkout_root="${nice_checkout_root:-$HOME/tvlc}"
depot_root=
if [ -f "$tvlc_root/depot_root" ]; then
depot_root="$(cat "$tvlc_root/depot_root")"
fi
if [ -d /depot ]; then
# don't require config on tvl nixos servers
depot_root="${depot_root:-/depot}"
fi
if [ -n "$depot_root" ]; then
export DEPOT_ROOT="$depot_root"
fi
if [ ! -d "$tvlc_root" ]; then
echo "tvlc: setup required"
echo "please run 'tvlc setup' from the depot root"
exit 1
fi

View file

@ -1,51 +0,0 @@
{ pkgs, depot, ... }:
let
pathScripts = pkgs.writeShellScript "imports" ''
export tvix_instantiate="${depot.third_party.nix}/bin/nix-instantiate"
export depot_scanner="${depot.tools.depot-scanner}/bin/depot-scanner"
'';
# setup: git rev-parse --show-toplevel > $tvlc_root/depot_root
# setup: mkdir $tvlc_root/clients
# setup: echo 1 > $tvlc_root/next_clientid
commonsh = pkgs.stdenv.mkDerivation {
name = "common.sh";
src = ./common.sh;
doCheck = true;
unpackPhase = "true";
buildPhase = ''
substitute ${./common.sh} $out --replace path-scripts ${pathScripts}
'';
checkPhase = ''
${pkgs.shellcheck}/bin/shellcheck $out ${pathScripts} && echo "SHELLCHECK OK"
'';
installPhase = ''
chmod +x $out
'';
};
tvlcNew = pkgs.stdenv.mkDerivation {
name = "tvlc-new";
src = ./tvlc-new;
doCheck = true;
unpackPhase = "true";
buildPhase = ''
substitute ${./tvlc-new} $out --replace common.sh ${commonsh}
'';
checkPhase = ''
${pkgs.shellcheck}/bin/shellcheck $out ${commonsh} ${pathScripts} && echo "SHELLCHECK OK"
'';
installPhase = ''
chmod +x $out
'';
};
in
{
inherit pathScripts;
inherit commonsh;
inherit tvlcNew;
}

View file

@ -1,103 +0,0 @@
#!/bin/bash
source common.sh
set -eu
set -o pipefail
function usage() {
echo "tvlc new [-n|--name CLIENTNAME] [derivation...]"
echo ""
cat <<EOF
The 'new' command creates a new git sparse checkout with the given name, and
contents needed to build the Nix derivation(s) specified on the command line.
Options:
-n/--name client-name: Sets the git branch and nice checkout name for the
workspace. If the option is not provided, the name will be based on the
first non-option command-line argument.
--branch branch-name: Sets the git branch name only.
EOF
}
checkout_name=
branch_name=
options=$(getopt -o 'n:' --long debug --long name: -- "$@")
eval set -- "$options"
while true; do
case "$1" in
-h)
usage
exit 0
;;
-v)
version
exit 0
;;
-n|--name)
shift
checkout_name="$1"
if [ -z "$branch_name" ]; then
branch_name=tvlc-"$1"
fi
;;
--branch)
shift
branch_name="$1"
;;
--)
shift
break
;;
esac
shift
done
if [ $# -eq 0 ]; then
echo "error: workspace name, target derivations required"
exit 1
fi
if [ -z "$checkout_name" ]; then
# TODO(riking): deduce
echo "error: workspace name (-n) required"
exit 1
fi
if [ -d "$nice_checkout_root/$checkout_name" ]; then
echo "error: checkout $checkout_name already exists"
# nb: shellescape checkout_name because we expect the user to copy-paste it
# shellcheck disable=SC1003
echo "consider deleting it with tvlc remove '${checkout_name/'/\'}'"
exit 1
fi
if [ -f "$DEPOT_ROOT/.git/refs/heads/$branch_name" ]; then
echo "error: branch $branch_name already exists in git"
# shellcheck disable=SC1003
echo "consider deleting it with cd $DEPOT_ROOT; git branch -d '${checkout_name/'/\'}'"
exit 1
fi
# The big one: call into Nix to figure out what paths the desired derivations depend on.
readarray -t includedPaths < <("$depot_scanner" --mode 'print' --only 'DEPOT' --relpath --depot "$DEPOT_ROOT" --nix-bin "$tvix_instantiate" "$@")
# bash math
checkout_id=$(("$(cat "$tvlc_root/next_clientid")"))
next_checkout_id=$(("$checkout_id"+1))
echo "$next_checkout_id" > "$tvlc_root/next_clientid"
checkout_dir="$tvlc_root/clients/$checkout_id"
mkdir "$checkout_dir"
cd "$DEPOT_ROOT"
git worktree add --no-checkout -b "$branch_name" "$checkout_dir"
# BUG: git not creating the /info/ subdir
mkdir "$DEPOT_ROOT/.git/worktrees/$checkout_id/info"
cd "$checkout_dir"
git sparse-checkout init --cone
git sparse-checkout set "${includedPaths[@]}"
ln -s "$checkout_dir" "$nice_checkout_root"/"$checkout_name"
echo "$nice_checkout_root/$checkout_name"