chore(tools): remove depot-scanner & tvlc
These are both unused things from a long time ago, which we don't need to keep around anymore. Their design doc has been marked as archived. Change-Id: Icd2744e511e78ec95ec8f39e5f79ed1fe98e9e4a Reviewed-on: https://cl.tvl.fyi/c/depot/+/7639 Autosubmit: tazjin <tazjin@tvl.su> Tested-by: BuildkiteCI Reviewed-by: flokli <flokli@flokli.de>
This commit is contained in:
Vincent Ambo
tazjin
parent
1e439fb5ad
commit
c6cb138565
10 changed files with 6 additions and 489 deletions
|
|
@ -1,3 +1,9 @@
|
|||
NOTE: This proposal is archived. We run `josh` instead, and long-term
|
||||
might want to integrate per-target dependency analysis with josh's
|
||||
workspace functionality.
|
||||
|
||||
-------------
|
||||
|
||||
Below is a prototype for a script to create Git sparse checkouts of the depot.
|
||||
The script below works today with relatively recent versions of git.
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
riking
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
{ depot, pkgs, ... }:
|
||||
|
||||
let
|
||||
localProto = depot.nix.buildGo.grpc {
|
||||
name = "code.tvl.fyi/tools/depot-scanner/proto";
|
||||
proto = ./depot_scanner.proto;
|
||||
};
|
||||
in
|
||||
depot.nix.buildGo.program
|
||||
{
|
||||
name = "depot-scanner";
|
||||
srcs = [
|
||||
./main.go
|
||||
];
|
||||
deps = [
|
||||
localProto
|
||||
];
|
||||
} // { inherit localProto; }
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
// Copyright 2020 TVL
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
syntax = "proto3";
|
||||
package tvl.tools.depot_scanner;
|
||||
option go_package = "code.tvl.fyi/tools/depot-scanner/proto";
|
||||
|
||||
enum PathType {
|
||||
UNKNOWN = 0;
|
||||
DEPOT = 1;
|
||||
STORE = 2;
|
||||
CORE = 3;
|
||||
}
|
||||
|
||||
message ScanRequest {
|
||||
// Which revision of the depot
|
||||
string revision = 1;
|
||||
string attr = 2;
|
||||
// Optionally, the attr to evaluate can be provided as a path to a folder or a
|
||||
// .nix file. This is used by the HTTP service.
|
||||
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
|
||||
string attrAsPath = 3;
|
||||
}
|
||||
|
||||
message ScanResponse {
|
||||
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
|
||||
repeated string depotPath = 1;
|
||||
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
|
||||
repeated string nixStorePath = 2;
|
||||
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
|
||||
repeated string corePkgsPath = 4;
|
||||
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
|
||||
repeated string otherPath = 3;
|
||||
|
||||
bytes derivation = 5;
|
||||
}
|
||||
|
||||
message ArchiveRequest {
|
||||
// buf:lint:ignore FIELD_LOWER_SNAKE_CASE
|
||||
repeated string depotPath = 1;
|
||||
}
|
||||
|
||||
message ArchiveChunk {
|
||||
bytes chunk = 1;
|
||||
}
|
||||
|
||||
service DepotScanService {
|
||||
rpc Scan(ScanRequest) returns (ScanResponse);
|
||||
|
||||
rpc MakeArchive(ArchiveRequest) returns (stream ArchiveChunk);
|
||||
}
|
||||
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
module code.tvl.fyi/tools/depot-scanner
|
||||
|
||||
go 1.14
|
||||
|
|
@ -1,227 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
|
||||
pb "code.tvl.fyi/tools/depot-scanner/proto"
|
||||
)
|
||||
|
||||
var nixInstantiatePath = flag.String("nix-bin", "/run/current-system/sw/bin/nix-instantiate", "path to nix-instantiate")
|
||||
var depotRoot = flag.String("depot", envOr("DEPOT_ROOT", "/depot/"), "path to tvl.fyi depot at current canon")
|
||||
var nixStoreRoot = flag.String("store-path", "/nix/store/", "prefix for all valid nix store paths")
|
||||
|
||||
var modeFlag = flag.String("mode", modeArchive, "operation mode. valid values: tar, print")
|
||||
var onlyFlag = flag.String("only", "", "only enable the listed output types, comma separated. valid values: DEPOT, STORE, CORE, UNKNOWN")
|
||||
var relativeFlag = flag.Bool("relpath", false, "when printing paths, print them relative to the root of their path type")
|
||||
|
||||
const (
|
||||
modeArchive = "tar"
|
||||
modePrint = "print"
|
||||
)
|
||||
|
||||
const (
|
||||
// String that identifies a path as belonging to nix corepkgs.
|
||||
corePkgsString = "/share/nix/corepkgs/"
|
||||
|
||||
depotTraceString = "trace: depot-scan: "
|
||||
)
|
||||
|
||||
type fileScanType int
|
||||
|
||||
const (
|
||||
unknownPath fileScanType = iota
|
||||
depotPath
|
||||
nixStorePath
|
||||
corePkgsPath
|
||||
)
|
||||
|
||||
func launchNix(attr string) (*exec.Cmd, io.ReadCloser, io.ReadCloser, error) {
|
||||
cmd := exec.Command(*nixInstantiatePath, "--trace-file-access", "-A", attr)
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
stdout.Close()
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
stdout.Close()
|
||||
stderr.Close()
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
return cmd, stdout, stderr, nil
|
||||
}
|
||||
|
||||
func categorizePath(path string) fileScanType {
|
||||
if strings.HasPrefix(path, *nixStoreRoot) {
|
||||
if strings.Contains(path, corePkgsString) {
|
||||
return corePkgsPath
|
||||
}
|
||||
return nixStorePath
|
||||
} else if strings.HasPrefix(path, *depotRoot) {
|
||||
return depotPath
|
||||
} else if strings.Contains(path, corePkgsString) {
|
||||
return corePkgsPath
|
||||
}
|
||||
return unknownPath
|
||||
}
|
||||
|
||||
func addPath(path string, out map[fileScanType]map[string]struct{}) {
|
||||
cat := categorizePath(path)
|
||||
if out[cat] == nil {
|
||||
out[cat] = make(map[string]struct{})
|
||||
}
|
||||
|
||||
out[cat][path] = struct{}{}
|
||||
}
|
||||
|
||||
func consumeOutput(stdout, stderr io.ReadCloser) (map[fileScanType]map[string]struct{}, string, error) {
|
||||
result := make(map[fileScanType]map[string]struct{})
|
||||
|
||||
scanner := bufio.NewScanner(stderr)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, depotTraceString) {
|
||||
addPath(strings.TrimPrefix(line, depotTraceString), result)
|
||||
} else {
|
||||
// print remaining stderr output of nix-instantiate
|
||||
// to prevent silent swallowing of possible important
|
||||
// error messages (e.g. about command line interface changes)
|
||||
fmt.Fprintf(os.Stderr, "nix-inst> %s\n", line)
|
||||
}
|
||||
}
|
||||
if scanner.Err() != nil {
|
||||
return nil, "", scanner.Err()
|
||||
}
|
||||
|
||||
// Get derivation path
|
||||
derivPath := ""
|
||||
scanner = bufio.NewScanner(stdout)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, *nixStoreRoot) {
|
||||
derivPath = line
|
||||
// consume the rest of the output
|
||||
}
|
||||
}
|
||||
if scanner.Err() != nil {
|
||||
return nil, "", scanner.Err()
|
||||
}
|
||||
|
||||
return result, derivPath, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
checkDepotRoot()
|
||||
|
||||
enabledPathTypes := make(map[pb.PathType]bool, 4)
|
||||
if len(*onlyFlag) > 0 {
|
||||
enabledOutputs := strings.Split(*onlyFlag, ",")
|
||||
for _, v := range enabledOutputs {
|
||||
i, ok := pb.PathType_value[strings.ToUpper(v)]
|
||||
if !ok {
|
||||
fmt.Fprintln(os.Stderr, "warning: unrecognized PathType name: ", v)
|
||||
continue
|
||||
}
|
||||
enabledPathTypes[pb.PathType(i)] = true
|
||||
}
|
||||
} else {
|
||||
// Default
|
||||
enabledPathTypes = map[pb.PathType]bool{
|
||||
pb.PathType_UNKNOWN: true,
|
||||
pb.PathType_DEPOT: true,
|
||||
pb.PathType_STORE: true,
|
||||
pb.PathType_CORE: true,
|
||||
}
|
||||
}
|
||||
|
||||
cmd, stdout, stderr, err := launchNix(flag.Arg(0))
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("could not launch nix: %w", err))
|
||||
}
|
||||
results, derivPath, err := consumeOutput(stdout, stderr)
|
||||
if err != nil {
|
||||
err2 := cmd.Wait()
|
||||
if err2 != nil {
|
||||
panic(fmt.Errorf("nix-instantiate failed: %w\nadditionally, while reading output: %w", err2, err))
|
||||
}
|
||||
panic(fmt.Errorf("problem reading nix output: %w", err))
|
||||
}
|
||||
err = cmd.Wait()
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("nix-instantiate failed: %w", err))
|
||||
}
|
||||
|
||||
_ = derivPath
|
||||
|
||||
if *modeFlag == "print" {
|
||||
if enabledPathTypes[pb.PathType_STORE] {
|
||||
for k, _ := range results[nixStorePath] {
|
||||
if *relativeFlag {
|
||||
k = strings.TrimPrefix(k, *nixStoreRoot)
|
||||
k = strings.TrimPrefix(k, "/")
|
||||
}
|
||||
fmt.Println(k)
|
||||
}
|
||||
}
|
||||
if enabledPathTypes[pb.PathType_DEPOT] {
|
||||
for k, _ := range results[depotPath] {
|
||||
if *relativeFlag {
|
||||
k = strings.TrimPrefix(k, *depotRoot)
|
||||
k = strings.TrimPrefix(k, "/")
|
||||
}
|
||||
fmt.Println(k)
|
||||
}
|
||||
}
|
||||
if enabledPathTypes[pb.PathType_CORE] {
|
||||
for k, _ := range results[corePkgsPath] {
|
||||
// TODO relativeFlag
|
||||
fmt.Println(k)
|
||||
}
|
||||
}
|
||||
if enabledPathTypes[pb.PathType_UNKNOWN] {
|
||||
for k, _ := range results[unknownPath] {
|
||||
fmt.Println(k)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
panic("unimplemented")
|
||||
}
|
||||
}
|
||||
|
||||
func envOr(envVar, def string) string {
|
||||
v := os.Getenv(envVar)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func checkDepotRoot() {
|
||||
if *depotRoot == "" {
|
||||
fmt.Fprintln(os.Stderr, "error: DEPOT_ROOT / -depot not set")
|
||||
os.Exit(2)
|
||||
}
|
||||
_, err := os.Stat(*depotRoot)
|
||||
if os.IsNotExist(err) {
|
||||
fmt.Fprintf(os.Stderr, "error: %q does not exist\ndid you forget to set DEPOT_ROOT / --depot ?\n", *depotRoot)
|
||||
os.Exit(1)
|
||||
} else if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: could not stat %q: %v\n", *depotRoot, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1 +0,0 @@
|
|||
riking
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
source path-scripts
|
||||
|
||||
XDG_DATA_HOME="${XDG_DATA_HOME:-$HOME/.local/share}"
|
||||
tvlc_root="$XDG_DATA_HOME/tvlc"
|
||||
|
||||
nice_checkout_root=
|
||||
if [ -f "$tvlc_root"/nice_checkout_root ]; then
|
||||
nice_checkout_root="$(cat "$tvlc_root"/nice_checkout_root)"
|
||||
fi
|
||||
nice_checkout_root="${nice_checkout_root:-$HOME/tvlc}"
|
||||
|
||||
depot_root=
|
||||
if [ -f "$tvlc_root/depot_root" ]; then
|
||||
depot_root="$(cat "$tvlc_root/depot_root")"
|
||||
fi
|
||||
if [ -d /depot ]; then
|
||||
# don't require config on tvl nixos servers
|
||||
depot_root="${depot_root:-/depot}"
|
||||
fi
|
||||
if [ -n "$depot_root" ]; then
|
||||
export DEPOT_ROOT="$depot_root"
|
||||
fi
|
||||
|
||||
if [ ! -d "$tvlc_root" ]; then
|
||||
echo "tvlc: setup required"
|
||||
echo "please run 'tvlc setup' from the depot root"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
{ pkgs, depot, ... }:
|
||||
|
||||
let
|
||||
pathScripts = pkgs.writeShellScript "imports" ''
|
||||
export tvix_instantiate="${depot.third_party.nix}/bin/nix-instantiate"
|
||||
export depot_scanner="${depot.tools.depot-scanner}/bin/depot-scanner"
|
||||
'';
|
||||
|
||||
# setup: git rev-parse --show-toplevel > $tvlc_root/depot_root
|
||||
# setup: mkdir $tvlc_root/clients
|
||||
# setup: echo 1 > $tvlc_root/next_clientid
|
||||
|
||||
commonsh = pkgs.stdenv.mkDerivation {
|
||||
name = "common.sh";
|
||||
src = ./common.sh;
|
||||
doCheck = true;
|
||||
unpackPhase = "true";
|
||||
buildPhase = ''
|
||||
substitute ${./common.sh} $out --replace path-scripts ${pathScripts}
|
||||
'';
|
||||
checkPhase = ''
|
||||
${pkgs.shellcheck}/bin/shellcheck $out ${pathScripts} && echo "SHELLCHECK OK"
|
||||
'';
|
||||
installPhase = ''
|
||||
chmod +x $out
|
||||
'';
|
||||
};
|
||||
|
||||
tvlcNew = pkgs.stdenv.mkDerivation {
|
||||
name = "tvlc-new";
|
||||
src = ./tvlc-new;
|
||||
doCheck = true;
|
||||
|
||||
unpackPhase = "true";
|
||||
buildPhase = ''
|
||||
substitute ${./tvlc-new} $out --replace common.sh ${commonsh}
|
||||
'';
|
||||
checkPhase = ''
|
||||
${pkgs.shellcheck}/bin/shellcheck $out ${commonsh} ${pathScripts} && echo "SHELLCHECK OK"
|
||||
'';
|
||||
installPhase = ''
|
||||
chmod +x $out
|
||||
'';
|
||||
};
|
||||
|
||||
in
|
||||
{
|
||||
inherit pathScripts;
|
||||
inherit commonsh;
|
||||
inherit tvlcNew;
|
||||
}
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
source common.sh
|
||||
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
function usage() {
|
||||
echo "tvlc new [-n|--name CLIENTNAME] [derivation...]"
|
||||
echo ""
|
||||
cat <<EOF
|
||||
The 'new' command creates a new git sparse checkout with the given name, and
|
||||
contents needed to build the Nix derivation(s) specified on the command line.
|
||||
|
||||
Options:
|
||||
-n/--name client-name: Sets the git branch and nice checkout name for the
|
||||
workspace. If the option is not provided, the name will be based on the
|
||||
first non-option command-line argument.
|
||||
--branch branch-name: Sets the git branch name only.
|
||||
EOF
|
||||
}
|
||||
|
||||
checkout_name=
|
||||
branch_name=
|
||||
|
||||
options=$(getopt -o 'n:' --long debug --long name: -- "$@")
|
||||
eval set -- "$options"
|
||||
while true; do
|
||||
case "$1" in
|
||||
-h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
-v)
|
||||
version
|
||||
exit 0
|
||||
;;
|
||||
-n|--name)
|
||||
shift
|
||||
checkout_name="$1"
|
||||
if [ -z "$branch_name" ]; then
|
||||
branch_name=tvlc-"$1"
|
||||
fi
|
||||
;;
|
||||
--branch)
|
||||
shift
|
||||
branch_name="$1"
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "error: workspace name, target derivations required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$checkout_name" ]; then
|
||||
# TODO(riking): deduce
|
||||
echo "error: workspace name (-n) required"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -d "$nice_checkout_root/$checkout_name" ]; then
|
||||
echo "error: checkout $checkout_name already exists"
|
||||
# nb: shellescape checkout_name because we expect the user to copy-paste it
|
||||
# shellcheck disable=SC1003
|
||||
echo "consider deleting it with tvlc remove '${checkout_name/'/\'}'"
|
||||
exit 1
|
||||
fi
|
||||
if [ -f "$DEPOT_ROOT/.git/refs/heads/$branch_name" ]; then
|
||||
echo "error: branch $branch_name already exists in git"
|
||||
# shellcheck disable=SC1003
|
||||
echo "consider deleting it with cd $DEPOT_ROOT; git branch -d '${checkout_name/'/\'}'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The big one: call into Nix to figure out what paths the desired derivations depend on.
|
||||
readarray -t includedPaths < <("$depot_scanner" --mode 'print' --only 'DEPOT' --relpath --depot "$DEPOT_ROOT" --nix-bin "$tvix_instantiate" "$@")
|
||||
|
||||
# bash math
|
||||
checkout_id=$(("$(cat "$tvlc_root/next_clientid")"))
|
||||
next_checkout_id=$(("$checkout_id"+1))
|
||||
echo "$next_checkout_id" > "$tvlc_root/next_clientid"
|
||||
|
||||
checkout_dir="$tvlc_root/clients/$checkout_id"
|
||||
mkdir "$checkout_dir"
|
||||
cd "$DEPOT_ROOT"
|
||||
git worktree add --no-checkout -b "$branch_name" "$checkout_dir"
|
||||
# BUG: git not creating the /info/ subdir
|
||||
mkdir "$DEPOT_ROOT/.git/worktrees/$checkout_id/info"
|
||||
|
||||
cd "$checkout_dir"
|
||||
git sparse-checkout init --cone
|
||||
git sparse-checkout set "${includedPaths[@]}"
|
||||
|
||||
ln -s "$checkout_dir" "$nice_checkout_root"/"$checkout_name"
|
||||
|
||||
echo "$nice_checkout_root/$checkout_name"
|
||||
Loading…
Reference in a new issue