From c6cb13856549907729e76035c818303f3c1fd244 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Mon, 26 Dec 2022 12:49:53 +0300 Subject: [PATCH] chore(tools): remove depot-scanner & tvlc These are both unused things from a long time ago, which we don't need to keep around anymore. Their design doc has been marked as archived. Change-Id: Icd2744e511e78ec95ec8f39e5f79ed1fe98e9e4a Reviewed-on: https://cl.tvl.fyi/c/depot/+/7639 Autosubmit: tazjin Tested-by: BuildkiteCI Reviewed-by: flokli --- docs/designs/SPARSE_CHECKOUTS.md | 6 + tools/depot-scanner/OWNERS | 1 - tools/depot-scanner/default.nix | 18 -- tools/depot-scanner/depot_scanner.proto | 52 ------ tools/depot-scanner/go.mod | 3 - tools/depot-scanner/main.go | 227 ------------------------ tools/tvlc/OWNERS | 1 - tools/tvlc/common.sh | 33 ---- tools/tvlc/default.nix | 51 ------ tools/tvlc/tvlc-new | 103 ----------- 10 files changed, 6 insertions(+), 489 deletions(-) delete mode 100644 tools/depot-scanner/OWNERS delete mode 100644 tools/depot-scanner/default.nix delete mode 100644 tools/depot-scanner/depot_scanner.proto delete mode 100644 tools/depot-scanner/go.mod delete mode 100644 tools/depot-scanner/main.go delete mode 100644 tools/tvlc/OWNERS delete mode 100644 tools/tvlc/common.sh delete mode 100644 tools/tvlc/default.nix delete mode 100755 tools/tvlc/tvlc-new diff --git a/docs/designs/SPARSE_CHECKOUTS.md b/docs/designs/SPARSE_CHECKOUTS.md index 7bd4963f6..820cb2c58 100644 --- a/docs/designs/SPARSE_CHECKOUTS.md +++ b/docs/designs/SPARSE_CHECKOUTS.md @@ -1,3 +1,9 @@ +NOTE: This proposal is archived. We run `josh` instead, and long-term +might want to integrate per-target dependency analysis with josh's +workspace functionality. + +------------- + Below is a prototype for a script to create Git sparse checkouts of the depot. The script below works today with relatively recent versions of git. diff --git a/tools/depot-scanner/OWNERS b/tools/depot-scanner/OWNERS deleted file mode 100644 index 34c668e2f..000000000 --- a/tools/depot-scanner/OWNERS +++ /dev/null @@ -1 +0,0 @@ -riking diff --git a/tools/depot-scanner/default.nix b/tools/depot-scanner/default.nix deleted file mode 100644 index 59b6e53f7..000000000 --- a/tools/depot-scanner/default.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ depot, pkgs, ... }: - -let - localProto = depot.nix.buildGo.grpc { - name = "code.tvl.fyi/tools/depot-scanner/proto"; - proto = ./depot_scanner.proto; - }; -in -depot.nix.buildGo.program - { - name = "depot-scanner"; - srcs = [ - ./main.go - ]; - deps = [ - localProto - ]; - } // { inherit localProto; } diff --git a/tools/depot-scanner/depot_scanner.proto b/tools/depot-scanner/depot_scanner.proto deleted file mode 100644 index ecb5b1cb3..000000000 --- a/tools/depot-scanner/depot_scanner.proto +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2020 TVL -// SPDX-License-Identifier: MIT - -syntax = "proto3"; -package tvl.tools.depot_scanner; -option go_package = "code.tvl.fyi/tools/depot-scanner/proto"; - -enum PathType { - UNKNOWN = 0; - DEPOT = 1; - STORE = 2; - CORE = 3; -} - -message ScanRequest { - // Which revision of the depot - string revision = 1; - string attr = 2; - // Optionally, the attr to evaluate can be provided as a path to a folder or a - // .nix file. This is used by the HTTP service. - // buf:lint:ignore FIELD_LOWER_SNAKE_CASE - string attrAsPath = 3; -} - -message ScanResponse { - // buf:lint:ignore FIELD_LOWER_SNAKE_CASE - repeated string depotPath = 1; - // buf:lint:ignore FIELD_LOWER_SNAKE_CASE - repeated string nixStorePath = 2; - // buf:lint:ignore FIELD_LOWER_SNAKE_CASE - repeated string corePkgsPath = 4; - // buf:lint:ignore FIELD_LOWER_SNAKE_CASE - repeated string otherPath = 3; - - bytes derivation = 5; -} - -message ArchiveRequest { - // buf:lint:ignore FIELD_LOWER_SNAKE_CASE - repeated string depotPath = 1; -} - -message ArchiveChunk { - bytes chunk = 1; -} - -service DepotScanService { - rpc Scan(ScanRequest) returns (ScanResponse); - - rpc MakeArchive(ArchiveRequest) returns (stream ArchiveChunk); -} - diff --git a/tools/depot-scanner/go.mod b/tools/depot-scanner/go.mod deleted file mode 100644 index bdd22fc1e..000000000 --- a/tools/depot-scanner/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module code.tvl.fyi/tools/depot-scanner - -go 1.14 diff --git a/tools/depot-scanner/main.go b/tools/depot-scanner/main.go deleted file mode 100644 index 9171587be..000000000 --- a/tools/depot-scanner/main.go +++ /dev/null @@ -1,227 +0,0 @@ -package main - -import ( - "bufio" - "flag" - "fmt" - "io" - "os" - "os/exec" - "strings" - - pb "code.tvl.fyi/tools/depot-scanner/proto" -) - -var nixInstantiatePath = flag.String("nix-bin", "/run/current-system/sw/bin/nix-instantiate", "path to nix-instantiate") -var depotRoot = flag.String("depot", envOr("DEPOT_ROOT", "/depot/"), "path to tvl.fyi depot at current canon") -var nixStoreRoot = flag.String("store-path", "/nix/store/", "prefix for all valid nix store paths") - -var modeFlag = flag.String("mode", modeArchive, "operation mode. valid values: tar, print") -var onlyFlag = flag.String("only", "", "only enable the listed output types, comma separated. valid values: DEPOT, STORE, CORE, UNKNOWN") -var relativeFlag = flag.Bool("relpath", false, "when printing paths, print them relative to the root of their path type") - -const ( - modeArchive = "tar" - modePrint = "print" -) - -const ( - // String that identifies a path as belonging to nix corepkgs. - corePkgsString = "/share/nix/corepkgs/" - - depotTraceString = "trace: depot-scan: " -) - -type fileScanType int - -const ( - unknownPath fileScanType = iota - depotPath - nixStorePath - corePkgsPath -) - -func launchNix(attr string) (*exec.Cmd, io.ReadCloser, io.ReadCloser, error) { - cmd := exec.Command(*nixInstantiatePath, "--trace-file-access", "-A", attr) - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, nil, nil, err - } - stderr, err := cmd.StderrPipe() - if err != nil { - stdout.Close() - return nil, nil, nil, err - } - - err = cmd.Start() - if err != nil { - stdout.Close() - stderr.Close() - return nil, nil, nil, err - } - - return cmd, stdout, stderr, nil -} - -func categorizePath(path string) fileScanType { - if strings.HasPrefix(path, *nixStoreRoot) { - if strings.Contains(path, corePkgsString) { - return corePkgsPath - } - return nixStorePath - } else if strings.HasPrefix(path, *depotRoot) { - return depotPath - } else if strings.Contains(path, corePkgsString) { - return corePkgsPath - } - return unknownPath -} - -func addPath(path string, out map[fileScanType]map[string]struct{}) { - cat := categorizePath(path) - if out[cat] == nil { - out[cat] = make(map[string]struct{}) - } - - out[cat][path] = struct{}{} -} - -func consumeOutput(stdout, stderr io.ReadCloser) (map[fileScanType]map[string]struct{}, string, error) { - result := make(map[fileScanType]map[string]struct{}) - - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, depotTraceString) { - addPath(strings.TrimPrefix(line, depotTraceString), result) - } else { - // print remaining stderr output of nix-instantiate - // to prevent silent swallowing of possible important - // error messages (e.g. about command line interface changes) - fmt.Fprintf(os.Stderr, "nix-inst> %s\n", line) - } - } - if scanner.Err() != nil { - return nil, "", scanner.Err() - } - - // Get derivation path - derivPath := "" - scanner = bufio.NewScanner(stdout) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, *nixStoreRoot) { - derivPath = line - // consume the rest of the output - } - } - if scanner.Err() != nil { - return nil, "", scanner.Err() - } - - return result, derivPath, nil -} - -func main() { - flag.Parse() - - checkDepotRoot() - - enabledPathTypes := make(map[pb.PathType]bool, 4) - if len(*onlyFlag) > 0 { - enabledOutputs := strings.Split(*onlyFlag, ",") - for _, v := range enabledOutputs { - i, ok := pb.PathType_value[strings.ToUpper(v)] - if !ok { - fmt.Fprintln(os.Stderr, "warning: unrecognized PathType name: ", v) - continue - } - enabledPathTypes[pb.PathType(i)] = true - } - } else { - // Default - enabledPathTypes = map[pb.PathType]bool{ - pb.PathType_UNKNOWN: true, - pb.PathType_DEPOT: true, - pb.PathType_STORE: true, - pb.PathType_CORE: true, - } - } - - cmd, stdout, stderr, err := launchNix(flag.Arg(0)) - if err != nil { - panic(fmt.Errorf("could not launch nix: %w", err)) - } - results, derivPath, err := consumeOutput(stdout, stderr) - if err != nil { - err2 := cmd.Wait() - if err2 != nil { - panic(fmt.Errorf("nix-instantiate failed: %w\nadditionally, while reading output: %w", err2, err)) - } - panic(fmt.Errorf("problem reading nix output: %w", err)) - } - err = cmd.Wait() - if err != nil { - panic(fmt.Errorf("nix-instantiate failed: %w", err)) - } - - _ = derivPath - - if *modeFlag == "print" { - if enabledPathTypes[pb.PathType_STORE] { - for k, _ := range results[nixStorePath] { - if *relativeFlag { - k = strings.TrimPrefix(k, *nixStoreRoot) - k = strings.TrimPrefix(k, "/") - } - fmt.Println(k) - } - } - if enabledPathTypes[pb.PathType_DEPOT] { - for k, _ := range results[depotPath] { - if *relativeFlag { - k = strings.TrimPrefix(k, *depotRoot) - k = strings.TrimPrefix(k, "/") - } - fmt.Println(k) - } - } - if enabledPathTypes[pb.PathType_CORE] { - for k, _ := range results[corePkgsPath] { - // TODO relativeFlag - fmt.Println(k) - } - } - if enabledPathTypes[pb.PathType_UNKNOWN] { - for k, _ := range results[unknownPath] { - fmt.Println(k) - } - } - } else { - panic("unimplemented") - } -} - -func envOr(envVar, def string) string { - v := os.Getenv(envVar) - if v == "" { - return def - } - return v -} - -func checkDepotRoot() { - if *depotRoot == "" { - fmt.Fprintln(os.Stderr, "error: DEPOT_ROOT / -depot not set") - os.Exit(2) - } - _, err := os.Stat(*depotRoot) - if os.IsNotExist(err) { - fmt.Fprintf(os.Stderr, "error: %q does not exist\ndid you forget to set DEPOT_ROOT / --depot ?\n", *depotRoot) - os.Exit(1) - } else if err != nil { - fmt.Fprintf(os.Stderr, "error: could not stat %q: %v\n", *depotRoot, err) - os.Exit(1) - } - -} diff --git a/tools/tvlc/OWNERS b/tools/tvlc/OWNERS deleted file mode 100644 index 34c668e2f..000000000 --- a/tools/tvlc/OWNERS +++ /dev/null @@ -1 +0,0 @@ -riking diff --git a/tools/tvlc/common.sh b/tools/tvlc/common.sh deleted file mode 100644 index fe7605857..000000000 --- a/tools/tvlc/common.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -set -eu -set -o pipefail - -source path-scripts - -XDG_DATA_HOME="${XDG_DATA_HOME:-$HOME/.local/share}" -tvlc_root="$XDG_DATA_HOME/tvlc" - -nice_checkout_root= -if [ -f "$tvlc_root"/nice_checkout_root ]; then - nice_checkout_root="$(cat "$tvlc_root"/nice_checkout_root)" -fi -nice_checkout_root="${nice_checkout_root:-$HOME/tvlc}" - -depot_root= -if [ -f "$tvlc_root/depot_root" ]; then - depot_root="$(cat "$tvlc_root/depot_root")" -fi -if [ -d /depot ]; then - # don't require config on tvl nixos servers - depot_root="${depot_root:-/depot}" -fi -if [ -n "$depot_root" ]; then - export DEPOT_ROOT="$depot_root" -fi - -if [ ! -d "$tvlc_root" ]; then - echo "tvlc: setup required" - echo "please run 'tvlc setup' from the depot root" - exit 1 -fi diff --git a/tools/tvlc/default.nix b/tools/tvlc/default.nix deleted file mode 100644 index a6f201485..000000000 --- a/tools/tvlc/default.nix +++ /dev/null @@ -1,51 +0,0 @@ -{ pkgs, depot, ... }: - -let - pathScripts = pkgs.writeShellScript "imports" '' - export tvix_instantiate="${depot.third_party.nix}/bin/nix-instantiate" - export depot_scanner="${depot.tools.depot-scanner}/bin/depot-scanner" - ''; - - # setup: git rev-parse --show-toplevel > $tvlc_root/depot_root - # setup: mkdir $tvlc_root/clients - # setup: echo 1 > $tvlc_root/next_clientid - - commonsh = pkgs.stdenv.mkDerivation { - name = "common.sh"; - src = ./common.sh; - doCheck = true; - unpackPhase = "true"; - buildPhase = '' - substitute ${./common.sh} $out --replace path-scripts ${pathScripts} - ''; - checkPhase = '' - ${pkgs.shellcheck}/bin/shellcheck $out ${pathScripts} && echo "SHELLCHECK OK" - ''; - installPhase = '' - chmod +x $out - ''; - }; - - tvlcNew = pkgs.stdenv.mkDerivation { - name = "tvlc-new"; - src = ./tvlc-new; - doCheck = true; - - unpackPhase = "true"; - buildPhase = '' - substitute ${./tvlc-new} $out --replace common.sh ${commonsh} - ''; - checkPhase = '' - ${pkgs.shellcheck}/bin/shellcheck $out ${commonsh} ${pathScripts} && echo "SHELLCHECK OK" - ''; - installPhase = '' - chmod +x $out - ''; - }; - -in -{ - inherit pathScripts; - inherit commonsh; - inherit tvlcNew; -} diff --git a/tools/tvlc/tvlc-new b/tools/tvlc/tvlc-new deleted file mode 100755 index 4ef0df5d3..000000000 --- a/tools/tvlc/tvlc-new +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -source common.sh - -set -eu -set -o pipefail - -function usage() { - echo "tvlc new [-n|--name CLIENTNAME] [derivation...]" - echo "" - cat < "$tvlc_root/next_clientid" - -checkout_dir="$tvlc_root/clients/$checkout_id" -mkdir "$checkout_dir" -cd "$DEPOT_ROOT" -git worktree add --no-checkout -b "$branch_name" "$checkout_dir" -# BUG: git not creating the /info/ subdir -mkdir "$DEPOT_ROOT/.git/worktrees/$checkout_id/info" - -cd "$checkout_dir" -git sparse-checkout init --cone -git sparse-checkout set "${includedPaths[@]}" - -ln -s "$checkout_dir" "$nice_checkout_root"/"$checkout_name" - -echo "$nice_checkout_root/$checkout_name"