Move closer to release

.github/workflows/manual.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - main
 jobs:
-  deploy:
+  deploy-unstable:
     runs-on: ubuntu-latest
 
     steps:
@@ -20,16 +20,36 @@ jobs:
           name: colmena
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
 
+      # == Manual
       - name: Build manual
         run: nix build -o out .#manual -L
 
       # Ugly hack so it has permission to delete the worktree afterwards
-      - name: Copy website
+      - name: Copy manual
         run: cp --no-preserve=mode -r out/ public/
 
-      - name: Deploy website
+      - name: Deploy manual
         uses: JamesIves/github-pages-deploy-action@4.1.5
         with:
           branch: gh-pages
           folder: public
           target-folder: unstable
+
+      # == Redirect Farm
+      # /future_api_version -> /unstable
+      - name: Check future API version
+        run: echo "api_version=${nix eval .#colmena.apiVersion}" >> $GITHUB_ENV
+
+      - name: Build redirect farm
+        run: nix build -o out .#manual.redirectFarm -L
+
+      # Ugly hack so it has permission to delete the worktree afterwards
+      - name: Copy redirect farm
+        run: cp --no-preserve=mode -r out/ redirect-farm/
+
+      - name: Deploy redirect farm
+        uses: JamesIves/github-pages-deploy-action@4.1.5
+        with:
+          branch: gh-pages
+          folder: redirect-farm
+          target-folder: '${{ env.api_version }}'

README.md

@@ -1,6 +1,6 @@
 # Colmena
 
-[![Manual](https://img.shields.io/badge/Manual-Pages-informational)](https://zhaofengli.github.io/colmena)
+[![Manual](https://img.shields.io/badge/Manual-Unstable-informational)](https://zhaofengli.github.io/colmena/unstable)
 [![Build](https://github.com/zhaofengli/colmena/workflows/Build/badge.svg)](https://github.com/zhaofengli/colmena/actions/workflows/build.yml)
 
 Colmena is a simple, stateless [NixOS](https://nixos.org) deployment tool modeled after [NixOps](https://github.com/NixOS/nixops) and [morph](https://github.com/DBCDK/morph), written in Rust.
@@ -22,20 +22,25 @@ $ <b>colmena apply --on @tag-a</b>
   <b>kappa</b> ✅ 2s Activation successful
 </pre>
 
-Colmena is still an early prototype.
-
 ## Installation
 
 Colmena doesn't have a stable release yet.
-To install the latest development version to the user profile, use `default.nix`:
+To install the latest development version to the user profile, use the following command:
 
+```bash
+nix-env -if https://github.com/zhaofengli/colmena/tarball/main
 ```
+
+Alternatively, if you have a local clone of the repo:
+
+```bash
 nix-env -if default.nix
 ```
 
-### Binary Cache
+### Unstable Binary Cache
 
 A public binary cache is available at https://colmena.cachix.org, courtesy of Cachix.
+This binary cache contains unstable versions of Colmena built by [GitHub Actions](https://github.com/zhaofengli/colmena/actions).
 
 ## Tutorial
 
@@ -179,11 +184,11 @@ Run `colmena build` in the same directory to build the configuration, or do `col
 
 Read [the Colmena Manual](https://zhaofengli.github.io/colmena).
 
-## Environment variables
+## Environment Variables
 
 - `SSH_CONFIG_FILE`: Path to a `ssh_config` file
 
-## Current limitations
+## Current Limitations
 
 - It's required to use SSH keys to log into the remote hosts, and interactive authentication will not work.
 - Error reporting is lacking.

default.nix

@@ -10,16 +10,19 @@ in {
   lib = pkgs.lib;
   stdenv = pkgs.stdenv;
   rustPlatform = pkgs.rustPlatform;
-in rustPlatform.buildRustPackage {
+in rustPlatform.buildRustPackage rec {
-  name = "colmena";
+  pname = "colmena";
   version = "0.2.0-pre";
 
+  # We guarantee CLI and Nix API stability for the same minor version
+  apiVersion = builtins.concatStringsSep "." (lib.take 2 (lib.splitString "." version));
+
   src = lib.cleanSourceWith {
     filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) [ "target" "manual" ]);
     src = lib.cleanSource ./.;
   };
 
-  cargoSha256 = "sha256-cpxvhP9TVEaIaiIZ+X22bDREqALpgWtW6koucVfMLwY=";
+  cargoSha256 = "sha256-IiAJ+sQasimcn4nSv4ACBwP1NLGNArtcIbwzkx0v/7w=";
 
   postInstall = lib.optionalString (stdenv.hostPlatform == stdenv.buildPlatform) ''
     mkdir completions

flake.nix

@@ -21,6 +21,7 @@
     packages = rec {
       colmena = import ./default.nix { inherit pkgs; };
 
+      # Full user manual
       manual = let
         colmena = self.packages.${system}.colmena;
         evalNix = import ./src/nix/eval.nix {
@@ -36,7 +37,11 @@
         inherit colmena deploymentOptionsMd metaOptionsMd;
       };
 
+      # User manual without the CLI reference
       manualFast = manual.override { colmena = null; };
+
+      # User manual with the version treated as stable
+      manualForceStable = manual.override { unstable = false; };
     };
 
     defaultApp = self.apps.${system}.colmena;
@@ -47,7 +52,7 @@
 
     devShell = pkgs.mkShell {
       inputsFrom = [ defaultPackage ];
-      nativeBuildInputs = with pkgs; [ mdbook ];
+      nativeBuildInputs = with pkgs; [ mdbook python3 ];
       shellHook = ''
         export NIX_PATH=nixpkgs=${pkgs.path}
       '';

manual/README.md

@@ -7,3 +7,27 @@ You can read the rendered version [here](https://zhaofengli.github.io/colmena).
 The manual is rendered using [mdBook](https://github.com/rust-lang/mdBook).
 To build the manual, do `nix build .#manual`.
 You can also do `nix build .#manualFast` for a version without the CLI usage reference.
+
+## Marking Text for Specific Versions
+
+You can mark text to be only visible in the unstable version of the manual:
+
+```
+<!-- UNSTABLE_BEGIN -->
+You are currently reading the unstable version of the Colmena Manual.
+Features described here will eventually become a part of version @apiVersion@.
+<!-- UNSTABLE_END -->
+```
+
+The opposite can be done with the `STABLE_{BEGIN,END}` markers:
+
+```
+<!-- STABLE_BEGIN -->
+You are currently reading the version @apiVersion@ of the Colmena Manual.
+<!-- STABLE_END -->
+```
+
+## Substitutions
+
+- `@version@` - Full version string
+- `@apiVersion@` - Stable API version string (major.minor)

manual/book.stable.toml

@@ -3,9 +3,13 @@ authors = ["Zhaofeng Li"]
 language = "en"
 multilingual = false
 src = "src"
-title = "Colmena RELEASE"
+title = "Colmena @apiVersion@"
 
 [output.html]
-site-url = "/colmena/RELEASE/"
+site-url = "/colmena/@apiVersion@/"
 git-repository-url = "https://github.com/zhaofengli/colmena"
 edit-url-template = "https://github.com/zhaofengli/colmena/edit/main/manual/{path}"
+
+[preprocessor.colmena]
+command = "python3 preprocess.py"
+renderer = [ "html", "epub" ]

manual/book.toml

@@ -9,3 +9,7 @@ title = "Colmena (Unstable)"
 site-url = "/colmena/unstable/"
 git-repository-url = "https://github.com/zhaofengli/colmena"
 edit-url-template = "https://github.com/zhaofengli/colmena/edit/main/manual/{path}"
+
+[preprocessor.colmena]
+command = "python3 preprocess.py"
+renderer = [ "html", "epub" ]

manual/default.nix

@@ -2,33 +2,66 @@ let
   notFound = typ: "<span style=\"color: red;\">Error: No ${typ} passed to the builder</span>";
 in
 
-{ stdenv, nix-gitignore, mdbook
+{ lib, stdenv, nix-gitignore, mdbook, python3, writeScript
-, release ? null
 , deploymentOptionsMd ? notFound "deployment options text"
 , metaOptionsMd ? notFound "meta options text"
 , colmena ? null
+
+, version ? null   # Full version (default: detected from colmena)
+, unstable ? null  # Whether this build is unstable (default: detected from version)
 }:
 
-stdenv.mkDerivation {
+let
-  inherit colmena deploymentOptionsMd metaOptionsMd release;
+  versionComp =
+    if version == null then
+      if colmena != null then colmena.version else "unstable"
+    else version;
 
-  name = if release == null then "colmena-manual-dev" else "colmena-manual-${release}";
+  unstableComp =
+    if unstable == null then versionComp == "unstable" || lib.hasInfix "-" versionComp
+    else unstable;
+
+  apiVersion = builtins.concatStringsSep "." (lib.take 2 (lib.splitString "." versionComp));
+
+  redirectTemplate = lib.escapeShellArg ''
+    <!doctype html>
+    <html>
+    <head>
+      <meta charset="utf-8">
+      <title>Redirecting</title>
+      <meta http-equiv="refresh" content="0; URL=https://zhaofengli.github.io/colmena@path@">
+    </head>
+    <body>
+      Redirecting to <a href="https://zhaofengli.github.io/colmena@path@">https://zhaofengli.github.io/colmena@path@</a>
+    </body>
+    </html>
+  '';
+
+in stdenv.mkDerivation {
+  inherit deploymentOptionsMd metaOptionsMd;
+
+  pname = "colmena-manual" + (if unstableComp then "-unstable" else "");
+  version = versionComp;
 
   src = nix-gitignore.gitignoreSource [] ./.;
 
-  nativeBuildInputs = [ mdbook ];
+  nativeBuildInputs = [ mdbook python3 ];
+
+  outputs = [ "out" "redirectFarm" ];
+
+  COLMENA_VERSION = versionComp;
+  COLMENA_UNSTABLE = unstableComp;
 
   patchPhase = ''
-    if [ -n "$release" ]; then
+    if [ -z "${toString unstableComp}" ]; then
-        find . -name '*.md' -exec sed -i "/REMOVE_FOR_RELEASE/d" {} \;
+        sed "s|@apiVersion@|${apiVersion}|g" book.stable.toml > book.toml
-        sed "s/RELEASE/$release/g" book.release.toml > book.toml
     fi
   '';
 
   buildPhase = ''
-    if [ -n "$colmena" ]; then
+    if [ -n "${colmena}" ]; then
         echo "Generating CLI help text"
-        $colmena/bin/colmena gen-help-markdown >> src/reference/cli.md
+        ${colmena}/bin/colmena gen-help-markdown >> src/reference/cli.md
     else
         echo "Error: No colmena executable passed to the builder" >> src/reference/cli.md
     fi
@@ -37,6 +70,26 @@ stdenv.mkDerivation {
     echo "$metaOptionsMd" >> src/reference/meta.md
 
     mdbook build -d $out
+
+    # Build the redirect farm
+    # GitHub Pages doesn't play well with directory symlinks. Default
+    # pages (index.html) don't work when a symlink is traversed.
+
+    mkdir -p $redirectFarm
+
+    subdir="/unstable"
+    if [ -z "${toString unstableComp}" ]; then
+      subdir="/${apiVersion}"
+    fi
+
+    pushd $redirectFarm
+    (cd $out; find . -name "*.html") | while read -r page; do
+        strippedPage=''${page#.}
+        target="$subdir$strippedPage"
+        mkdir -p $(dirname $page)
+        echo ${redirectTemplate} | sed "s|@path@|$target|g" > $page
+    done
+    popd
   '';
 
   installPhase = "true";

manual/preprocess.py

@@ -0,0 +1,72 @@
+# Markdown preprocessor
+#
+# 1. Removes marked blocks from markdown files
+#    (unstable-only text from stable versions and vice versa)
+# 2. Substitutes @version@ in text with the full version
+# 3. Substitutes @apiVersion@ in text with major.minor
+#
+# Environment:
+# - COLMENA_VERSION=${fullVersion}
+# - COLMENA_UNSTABLE="1" or unset
+
+import json
+import os
+import pprint
+import re
+import sys
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+def process(book, version, unstable):
+    marker_to_remove = "STABLE" if unstable else "UNSTABLE"
+
+    api_version = re.match("^[0-9]+\.[0-9]+(?=$|\.[0-9]+(.*)?$)", version)
+    if api_version:
+        api_version = api_version.group()
+    else:
+        api_version = version
+
+    version_debug = f"{version} (apiVersion={api_version}, unstable={str(unstable)})"
+
+    def replace_version_markers(s):
+        s = s.replace("@version@", version)
+        s = s.replace("@apiVersion@", api_version)
+        return s
+
+    def process_item(item):
+        chapter = item["Chapter"]
+
+        for sub_item in chapter["sub_items"]:
+            process_item(sub_item)
+
+        regex = r".*\b{marker}_BEGIN\b(.|\n|\r)*?\b{marker}_END\b.*".format(marker=marker_to_remove)
+
+        chapter["content"] = f"<!-- Generated from version {version_debug} -->\n" + chapter["content"]
+        chapter["content"] = re.sub(regex, "", chapter["content"])
+        chapter["content"] = replace_version_markers(chapter["content"])
+
+        #eprint(f"Processed {chapter['name']}")
+
+    eprint(f"Version is {version_debug}")
+
+    for section in book['sections']:
+        process_item(section)
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "supports":
+            #eprint("mdbook asked about support for", sys.argv[2])
+            sys.exit(0)
+
+    version = os.environ.get("COLMENA_VERSION", "unstable")
+    unstable = bool(os.environ.get("COLMENA_UNSTABLE", ""))
+
+    if version in [ "" "unstable" ]:
+        unstable = True
+
+    context, book = json.load(sys.stdin)
+
+    process(book, version, unstable)
+
+    print(json.dumps(book))

manual/src/introduction.md

@@ -18,7 +18,14 @@ Interested? Get started [here](tutorial)!
   <b>kappa</b> ✅ 2s Activation successful
 </div></pre>
 
-You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena).<!-- REMOVE_FOR_RELEASE -->
+<!-- UNSTABLE_BEGIN -->
+You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena).
+Features described here will eventually become a part of **version @apiVersion@**.
+<!-- UNSTABLE_END -->
+
+<!-- STABLE_BEGIN -->
+You are currently reading **version @apiVersion@** of the Colmena Manual, built against version @version@.
+<!-- STABLE_END -->
 
 ## Links
 

manual/src/reference/cli.md

@@ -1,7 +1,16 @@
 # Command Line Arguments
 
-You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). <!-- REMOVE_FOR_RELEASE -->
+<!-- UNSTABLE_BEGIN -->
+You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena).
+<!-- UNSTABLE_END -->
 
-The following are the auto-generated help messages that are printed when you invoke any sub-command with `--help`:
+The following are the help messages that will be printed when you invoke any sub-command with `--help`:
 
-<!-- The following is injected by the build system -->
+<!--
+    The following is injected by the build system
+
+    Looking to improve the help messages? They are located in:
+
+    - src/cli.rs
+    - src/command/<subcommand>.rs
+-->

manual/src/reference/deployment.md

@@ -1,7 +1,13 @@
 # Deployment Options
 
-You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). <!-- REMOVE_FOR_RELEASE -->
+<!-- UNSTABLE_BEGIN -->
+You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena).
+<!-- UNSTABLE_END -->
 
 Colmena adds a set of extra options that can be used in your NixOS configurations under the `deployment` prefix.
 
-<!-- The following is injected by the build system -->
+<!--
+    The following is injected by the build system
+
+    Looking to improve the description? Head to `src/nix/eval.nix` in the repo :)
+-->

manual/src/reference/index.md

@@ -1,6 +1,8 @@
 # Reference
 
-You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). Go back to [the home page](../introduction.md) for links to other versions of the manual. <!-- REMOVE_FOR_RELEASE --> 
+<!-- UNSTABLE_BEGIN -->
+You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena).
+<!-- UNSTABLE_END -->
 
 This section contains detailed listings of options and parameters accepted by Colmena:
 

manual/src/reference/meta.md

@@ -1,10 +1,16 @@
 # Meta Options
 
-You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). <!-- REMOVE_FOR_RELEASE -->
+<!-- UNSTABLE_BEGIN -->
+You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena).
+<!-- UNSTABLE_END -->
 
 The following is a list of options that can be added to the `meta` attribute set.
 
 For compatibility with NixOps, you may name it `network` instead of `meta`.
 However, you cannot specify both at the same time.
 
-<!-- The following is injected by the build system -->
+<!--
+    The following is injected by the build system
+
+    Looking to improve the descriptions? Head to `src/nix/eval.nix` in the repo :)
+-->

manual/src/tutorial/flakes.md

@@ -2,21 +2,35 @@
 
 ## Installation
 
+<!-- STABLE_BEGIN -->
 Colmena doesn't have a stable release yet.
-To quickly try Colmena out, use the following command to enter an ephemeral environment with `colmena`:
 
-```console
+If you are interested in trying out the bleeding-edge version of Colmena, Read [the unstable version](https://zhaofengli.github.io/colmena/unstable) of the Manual for instructions.
-$ nix shell github:zhaofengli/colmena
+<!-- STABLE_END -->
+
+<!-- UNSTABLE_BEGIN -->
+<!-- To install the latest stable version, read [the corresponding Manual](https://zhaofengli.github.io/colmena/stable) for instructions. -->
+
+To quickly try Colmena out, use the following command to enter an ephemeral environment with the latest development version of `colmena`:
+
+```bash
+nix shell github:zhaofengli/colmena
 ```
 
-To install the latest development version to the user profile, use the following command:
+To install Colmena to the user profile, use the following command:
 
-```console
+```bash
-$ nix-env -if https://github.com/zhaofengli/colmena/tarball/main
+nix-env -if https://github.com/zhaofengli/colmena/tarball/main
 ```
 
 You can also add `github:zhaofengli/colmena` as an input in your Flake and add the `colmena` package to your `devShell`.
 
+### Unstable Binary Cache
+
+A public binary cache is available at [https://colmena.cachix.org](https://colmena.cachix.org), courtesy of Cachix.
+This binary cache contains unstable versions of Colmena built by [GitHub Actions](https://github.com/zhaofengli/colmena/actions).
+<!-- UNSTABLE_END -->
+
 ## Basic Configuration
 
 Colmena reads the `colmena` output in your Flake.
@@ -55,18 +69,18 @@ Here is a short example:
 ```
 
 The full set of `deployment` options can be found [here](../reference/deployment.md).
-For some inspiration, you can check out the short example in [the main tutorial](index.md).
+You can also check out the example in [the main tutorial](index.md) for some inspiration.
 
 Now you are ready to use Colmena! To build the configuration:
 
-```console
+```bash
-$ colmena build
+colmena build
 ```
 
 To build and deploy to all nodes:
 
-```console
+```bash
-$ colmena apply
+colmena apply
 ```
 
 ## Next Steps

manual/src/tutorial/index.md

@@ -2,13 +2,27 @@
 
 ## Installation
 
+<!-- STABLE_BEGIN -->
 Colmena doesn't have a stable release yet.
+
+If you are interested in trying out the bleeding-edge version of Colmena, Read [the unstable version](https://zhaofengli.github.io/colmena/unstable) of the Manual for instructions.
+<!-- STABLE_END -->
+
+<!-- UNSTABLE_BEGIN -->
+<!-- To install the latest stable version, read [the corresponding Manual](https://zhaofengli.github.io/colmena/stable) for instructions. -->
+
 To install the latest development version to the user profile, use the following command:
 
-```console
+```bash
-$ nix-env -if https://github.com/zhaofengli/colmena/tarball/main
+nix-env -if https://github.com/zhaofengli/colmena/tarball/main
 ```
 
+### Unstable Binary Cache
+
+A public binary cache is available at [https://colmena.cachix.org](https://colmena.cachix.org), courtesy of Cachix.
+This binary cache contains unstable versions of Colmena built by [GitHub Actions](https://github.com/zhaofengli/colmena/actions).
+<!-- UNSTABLE_END -->
+
 ## Basic Configuration
 
 *If you use Nix Flakes, follow the Flake version [here](flakes.md).*
@@ -108,14 +122,14 @@ The full set of `deployment` options can be found [here](../reference/deployment
 
 Now you are ready to use Colmena! To build the configuration:
 
-```console
+```bash
-$ colmena build
+colmena build
 ```
 
 To build and deploy to all nodes:
 
-```console
+```bash
-$ colmena apply
+colmena apply
 ```
 
 ## Next Steps