mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(popcount): Cache seen narinfos on disk

Browse source
This commit is contained in:
Vincent Ambo 2019-10-31 17:48:56 +00:00 committed by Vincent Ambo
parent 6a2fb092a7
commit 05b5b1718a
1 changed files with 14 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
tools/nixery/popcount/popcount.go
View file

@ -160,6 +160,11 @@ func narInfoToRefs(narinfo string) []string {
} }
func fetchNarInfo(i *item) (string, error) { func fetchNarInfo(i *item) (string, error) {
file, err := ioutil.ReadFile("popcache/" + i.hash)
if err == nil {
return string(file), nil
}
resp, err := client.Get(fmt.Sprintf("https://cache.nixos.org/%s.narinfo", i.hash)) resp, err := client.Get(fmt.Sprintf("https://cache.nixos.org/%s.narinfo", i.hash))
if err != nil { if err != nil {
return "", err return "", err
@ -168,6 +173,10 @@ func fetchNarInfo(i *item) (string, error) {
defer resp.Body.Close() defer resp.Body.Close()
narinfo, err := ioutil.ReadAll(resp.Body) narinfo, err := ioutil.ReadAll(resp.Body)
// best-effort write the file to the cache
ioutil.WriteFile("popcache/" + i.hash, narinfo, 0644)
return string(narinfo), err return string(narinfo), err
} }
@ -208,6 +217,11 @@ func main() {
log.Fatalf("Nix channel must be specified as first argument") log.Fatalf("Nix channel must be specified as first argument")
} }
err := os.MkdirAll("popcache", 0755)
if err != nil {
log.Fatalf("Failed to create 'popcache' directory in current folder: %s\n", err)
}
count := 42 // concurrent downloader count count := 42 // concurrent downloader count
channel := os.Args[1] channel := os.Args[1]
log.Printf("Fetching metadata for channel '%s'\n", channel) log.Printf("Fetching metadata for channel '%s'\n", channel)