diff --git a/.forgejo/workflows/eval.yaml b/.forgejo/workflows/eval.yaml index d1628b4..90c06e0 100644 --- a/.forgejo/workflows/eval.yaml +++ b/.forgejo/workflows/eval.yaml @@ -69,20 +69,139 @@ jobs: # Enter the shell nix-shell --run 'colmena build --on rescue01' - push_to_cache: + build_geo01: + runs-on: nix + steps: + - uses: actions/checkout@v3 + + - name: Build geo01 + run: | + # Enter the shell + nix-shell --run 'colmena build --on geo01' + + build_geo02: + runs-on: nix + steps: + - uses: actions/checkout@v3 + + - name: Build geo02 + run: | + # Enter the shell + nix-shell --run 'colmena build --on geo02' + + push_to_cache_compute01: runs-on: nix needs: - build_compute01 + steps: + - uses: actions/checkout@v3 + + - name: Push to cache + run: nix-shell --run push-to-nix-cache + env: + STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/" + STORE_USER: "admin" + STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }} + NODES: '[ "compute01" ]' + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: outputs_compute01 + path: uploaded.txt + + push_to_cache_storage01: + runs-on: nix + needs: - build_storage01 - - build_vault01 - - build_web01 - - build_web02 + steps: + - uses: actions/checkout@v3 + + - name: Push to cache + run: nix-shell --run push-to-nix-cache + env: + STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/" + STORE_USER: "admin" + STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }} + NODES: '[ "storage01" ]' + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: outputs_storage01 + path: uploaded.txt + + push_to_cache_rescue01: + runs-on: nix + needs: - build_rescue01 steps: - uses: actions/checkout@v3 - name: Push to cache - run: nix-shell --run push-to-cache + run: nix-shell --run push-to-nix-cache env: - ATTIC_ENDPOINT: "https://cachix.dgnum.eu" - ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/" + STORE_USER: "admin" + STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }} + NODES: '[ "rescue01" ]' + + push_to_cache_geo01: + runs-on: nix + needs: + - build_geo01 + steps: + - uses: actions/checkout@v3 + + - name: Push to cache + run: nix-shell --run push-to-nix-cache + env: + STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/" + STORE_USER: "admin" + STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }} + NODES: '[ "geo01" ]' + + push_to_cache_geo02: + runs-on: nix + needs: + - build_geo02 + steps: + - uses: actions/checkout@v3 + + - name: Push to cache + run: nix-shell --run push-to-nix-cache + env: + STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/" + STORE_USER: "admin" + STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }} + NODES: '[ "geo02" ]' + + push_to_cache_web01: + runs-on: nix + needs: + - build_web01 + steps: + - uses: actions/checkout@v3 + + - name: Push to cache + run: nix-shell --run push-to-nix-cache + env: + STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/" + STORE_USER: "admin" + STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }} + NODES: '[ "web01" ]' + + push_to_cache_web02: + runs-on: nix + needs: + - build_web02 + steps: + - uses: actions/checkout@v3 + + - name: Push to cache + run: nix-shell --run push-to-nix-cache + env: + STORE_ENDPOINT: "https://tvix-store.dgnum.eu/infra-signing/" + STORE_USER: "admin" + STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }} + NODES: '[ "web02" ]' diff --git a/scripts/default.nix b/scripts/default.nix index 84945a0..8f31129 100644 --- a/scripts/default.nix +++ b/scripts/default.nix @@ -34,6 +34,7 @@ let "launch-vm" "list-nodes" "push-to-cache" + "push-to-nix-cache" "cache" ]; in diff --git a/scripts/push-to-cache.sh b/scripts/push-to-cache.sh index b6d1ae3..a753afd 100755 --- a/scripts/push-to-cache.sh +++ b/scripts/push-to-cache.sh @@ -8,6 +8,6 @@ ENDPOINT=${ATTIC_ENDPOINT:-https://cachix.dgnum.eu} @colmena@/bin/colmena eval -E '{ nodes, lib, ... }: lib.mapAttrsToList (_: v: v.config.system.build.toplevel.drvPath) nodes' |\ @jq@/bin/jq -r '.[]' |\ -xargs nix-store -q -R --include-outputs |\ +xargs -n 10 nix-store -q -R --include-outputs |\ sed '/\.drv$/d' |\ xargs @attic@/bin/attic push dgnum:infra diff --git a/scripts/push-to-nix-cache.sh b/scripts/push-to-nix-cache.sh new file mode 100755 index 0000000..f537d4d --- /dev/null +++ b/scripts/push-to-nix-cache.sh @@ -0,0 +1,20 @@ +set -e +set -u +set -o pipefail + +ENDPOINT=${STORE_ENDPOINT:-https://tvix-cache.dgnum.eu/infra-singing/} + +cat > .netrc << EOF +default +login $STORE_USER +password $STORE_PASSWORD +EOF + +@colmena@/bin/colmena eval -E "{ nodes, lib, ... }: builtins.map (v: nodes.\${v}.config.system.build.toplevel.drvPath) ${NODES:-(builtins.attrNames nodes)}" |\ +@jq@/bin/jq -r '.[]' |\ +xargs nix-store -q -R --include-outputs |\ +sed '/\.drv$/d' |\ +tee uploaded.txt |\ +xargs nix copy --to "$ENDPOINT?compression=none" --extra-experimental-features nix-command --netrc-file ./.netrc + +rm .netrc