refactor(server): Expose layer grouping logic via a function
Refactors the layer grouping package (which previously compiled to a separate binary) to expose the layer grouping logic via a function instead. This is the next step towards creating layers inside of the server component instead of in Nix. Relates to #50.
This commit is contained in:
parent
8c79d085ae
commit
9c3c622403
1 changed files with 24 additions and 55 deletions
|
@ -1,6 +1,6 @@
|
|||
// This program reads an export reference graph (i.e. a graph representing the
|
||||
// runtime dependencies of a set of derivations) created by Nix and groups them
|
||||
// in a way that is likely to match the grouping for other derivation sets with
|
||||
// This package reads an export reference graph (i.e. a graph representing the
|
||||
// runtime dependencies of a set of derivations) created by Nix and groups it in
|
||||
// a way that is likely to match the grouping for other derivation sets with
|
||||
// overlapping dependencies.
|
||||
//
|
||||
// This is used to determine which derivations to include in which layers of a
|
||||
|
@ -9,8 +9,8 @@
|
|||
// # Inputs
|
||||
//
|
||||
// * a graph of Nix runtime dependencies, generated via exportReferenceGraph
|
||||
// * a file containing absolute popularity values of packages in the
|
||||
// Nix package set (in the form of a direct reference count)
|
||||
// * popularity values of each package in the Nix package set (in the form of a
|
||||
// direct reference count)
|
||||
// * a maximum number of layers to allocate for the image (the "layer budget")
|
||||
//
|
||||
// # Algorithm
|
||||
|
@ -103,9 +103,6 @@
|
|||
package layers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"regexp"
|
||||
"sort"
|
||||
|
@ -114,9 +111,11 @@ import (
|
|||
"gonum.org/v1/gonum/graph/simple"
|
||||
)
|
||||
|
||||
// closureGraph represents the structured attributes Nix outputs when asking it
|
||||
// for the exportReferencesGraph of a list of derivations.
|
||||
type exportReferences struct {
|
||||
// RuntimeGraph represents structured information from Nix about the runtime
|
||||
// dependencies of a derivation.
|
||||
//
|
||||
// This is generated in Nix by using the exportReferencesGraph feature.
|
||||
type RuntimeGraph struct {
|
||||
References struct {
|
||||
Graph []string `json:"graph"`
|
||||
} `json:"exportReferencesGraph"`
|
||||
|
@ -135,14 +134,14 @@ type exportReferences struct {
|
|||
// of the nixpkgs tree.
|
||||
type Popularity = map[string]int
|
||||
|
||||
// layer represents the data returned for each layer that Nix should
|
||||
// Layer represents the data returned for each layer that Nix should
|
||||
// build for the container image.
|
||||
type layer struct {
|
||||
type Layer struct {
|
||||
Contents []string `json:"contents"`
|
||||
mergeRating uint64
|
||||
}
|
||||
|
||||
func (a layer) merge(b layer) layer {
|
||||
func (a Layer) merge(b Layer) Layer {
|
||||
a.Contents = append(a.Contents, b.Contents...)
|
||||
a.mergeRating += b.mergeRating
|
||||
return a
|
||||
|
@ -209,7 +208,7 @@ func insertEdges(graph *simple.DirectedGraph, cmap *map[string]*closure, node *c
|
|||
}
|
||||
|
||||
// Create a graph structure from the references supplied by Nix.
|
||||
func buildGraph(refs *exportReferences, pop *Popularity) *simple.DirectedGraph {
|
||||
func buildGraph(refs *RuntimeGraph, pop *Popularity) *simple.DirectedGraph {
|
||||
cmap := make(map[string]*closure)
|
||||
graph := simple.NewDirectedGraph()
|
||||
|
||||
|
@ -259,7 +258,7 @@ func buildGraph(refs *exportReferences, pop *Popularity) *simple.DirectedGraph {
|
|||
// Extracts a subgraph starting at the specified root from the
|
||||
// dominator tree. The subgraph is converted into a flat list of
|
||||
// layers, each containing the store paths and merge rating.
|
||||
func groupLayer(dt *flow.DominatorTree, root *closure) layer {
|
||||
func groupLayer(dt *flow.DominatorTree, root *closure) Layer {
|
||||
size := root.Size
|
||||
contents := []string{root.Path}
|
||||
children := dt.DominatedBy(root.ID())
|
||||
|
@ -273,7 +272,7 @@ func groupLayer(dt *flow.DominatorTree, root *closure) layer {
|
|||
children = append(children, dt.DominatedBy(child.ID())...)
|
||||
}
|
||||
|
||||
return layer{
|
||||
return Layer{
|
||||
Contents: contents,
|
||||
// TODO(tazjin): The point of this is to factor in
|
||||
// both the size and the popularity when making merge
|
||||
|
@ -288,10 +287,10 @@ func groupLayer(dt *flow.DominatorTree, root *closure) layer {
|
|||
//
|
||||
// Layers are merged together until they fit into the layer budget,
|
||||
// based on their merge rating.
|
||||
func dominate(budget int, graph *simple.DirectedGraph) []layer {
|
||||
func dominate(budget int, graph *simple.DirectedGraph) []Layer {
|
||||
dt := flow.Dominators(graph.Node(0), graph)
|
||||
|
||||
var layers []layer
|
||||
var layers []Layer
|
||||
for _, n := range dt.DominatedBy(dt.Root().ID()) {
|
||||
layers = append(layers, groupLayer(&dt, n.(*closure)))
|
||||
}
|
||||
|
@ -313,40 +312,10 @@ func dominate(budget int, graph *simple.DirectedGraph) []layer {
|
|||
return layers
|
||||
}
|
||||
|
||||
func main() {
|
||||
graphFile := flag.String("graph", ".attrs.json", "Input file containing graph")
|
||||
popFile := flag.String("pop", "popularity.json", "Package popularity data")
|
||||
outFile := flag.String("out", "layers.json", "File to write layers to")
|
||||
layerBudget := flag.Int("budget", 94, "Total layer budget available")
|
||||
flag.Parse()
|
||||
|
||||
// Parse graph data
|
||||
file, err := ioutil.ReadFile(*graphFile)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load input: %s\n", err)
|
||||
}
|
||||
|
||||
var refs exportReferences
|
||||
err = json.Unmarshal(file, &refs)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to deserialise input: %s\n", err)
|
||||
}
|
||||
|
||||
// Parse popularity data
|
||||
popBytes, err := ioutil.ReadFile(*popFile)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load input: %s\n", err)
|
||||
}
|
||||
|
||||
var pop Popularity
|
||||
err = json.Unmarshal(popBytes, &pop)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to deserialise input: %s\n", err)
|
||||
}
|
||||
|
||||
graph := buildGraph(&refs, &pop)
|
||||
layers := dominate(*layerBudget, graph)
|
||||
|
||||
j, _ := json.Marshal(layers)
|
||||
ioutil.WriteFile(*outFile, j, 0644)
|
||||
// GroupLayers applies the algorithm described above the its input and returns a
|
||||
// list of layers, each consisting of a list of Nix store paths that it should
|
||||
// contain.
|
||||
func GroupLayers(refs *RuntimeGraph, pop *Popularity, budget int) []Layer {
|
||||
graph := buildGraph(refs, pop)
|
||||
return dominate(budget, graph)
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue