diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index 7edacbc..47608c5 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -5,7 +5,7 @@ on: branches: - main jobs: - deploy: + deploy-unstable: runs-on: ubuntu-latest steps: @@ -20,16 +20,36 @@ jobs: name: colmena authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + # == Manual - name: Build manual run: nix build -o out .#manual -L # Ugly hack so it has permission to delete the worktree afterwards - - name: Copy website + - name: Copy manual run: cp --no-preserve=mode -r out/ public/ - - name: Deploy website + - name: Deploy manual uses: JamesIves/github-pages-deploy-action@4.1.5 with: branch: gh-pages folder: public target-folder: unstable + + # == Redirect Farm + # /future_api_version -> /unstable + - name: Check future API version + run: echo "api_version=${nix eval .#colmena.apiVersion}" >> $GITHUB_ENV + + - name: Build redirect farm + run: nix build -o out .#manual.redirectFarm -L + + # Ugly hack so it has permission to delete the worktree afterwards + - name: Copy redirect farm + run: cp --no-preserve=mode -r out/ redirect-farm/ + + - name: Deploy redirect farm + uses: JamesIves/github-pages-deploy-action@4.1.5 + with: + branch: gh-pages + folder: redirect-farm + target-folder: '${{ env.api_version }}' diff --git a/README.md b/README.md index cda5fca..61f775d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Colmena -[![Manual](https://img.shields.io/badge/Manual-Pages-informational)](https://zhaofengli.github.io/colmena) +[![Manual](https://img.shields.io/badge/Manual-Unstable-informational)](https://zhaofengli.github.io/colmena/unstable) [![Build](https://github.com/zhaofengli/colmena/workflows/Build/badge.svg)](https://github.com/zhaofengli/colmena/actions/workflows/build.yml) Colmena is a simple, stateless [NixOS](https://nixos.org) deployment tool modeled after [NixOps](https://github.com/NixOS/nixops) and [morph](https://github.com/DBCDK/morph), written in Rust. @@ -22,20 +22,25 @@ $ colmena apply --on @tag-a kappa ✅ 2s Activation successful -Colmena is still an early prototype. - ## Installation Colmena doesn't have a stable release yet. -To install the latest development version to the user profile, use `default.nix`: +To install the latest development version to the user profile, use the following command: +```bash +nix-env -if https://github.com/zhaofengli/colmena/tarball/main ``` + +Alternatively, if you have a local clone of the repo: + +```bash nix-env -if default.nix ``` -### Binary Cache +### Unstable Binary Cache A public binary cache is available at https://colmena.cachix.org, courtesy of Cachix. +This binary cache contains unstable versions of Colmena built by [GitHub Actions](https://github.com/zhaofengli/colmena/actions). ## Tutorial @@ -179,11 +184,11 @@ Run `colmena build` in the same directory to build the configuration, or do `col Read [the Colmena Manual](https://zhaofengli.github.io/colmena). -## Environment variables +## Environment Variables - `SSH_CONFIG_FILE`: Path to a `ssh_config` file -## Current limitations +## Current Limitations - It's required to use SSH keys to log into the remote hosts, and interactive authentication will not work. - Error reporting is lacking. diff --git a/default.nix b/default.nix index b92ad2d..ce45229 100644 --- a/default.nix +++ b/default.nix @@ -10,16 +10,19 @@ in { lib = pkgs.lib; stdenv = pkgs.stdenv; rustPlatform = pkgs.rustPlatform; -in rustPlatform.buildRustPackage { - name = "colmena"; +in rustPlatform.buildRustPackage rec { + pname = "colmena"; version = "0.2.0-pre"; + # We guarantee CLI and Nix API stability for the same minor version + apiVersion = builtins.concatStringsSep "." (lib.take 2 (lib.splitString "." version)); + src = lib.cleanSourceWith { filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) [ "target" "manual" ]); src = lib.cleanSource ./.; }; - cargoSha256 = "sha256-cpxvhP9TVEaIaiIZ+X22bDREqALpgWtW6koucVfMLwY="; + cargoSha256 = "sha256-IiAJ+sQasimcn4nSv4ACBwP1NLGNArtcIbwzkx0v/7w="; postInstall = lib.optionalString (stdenv.hostPlatform == stdenv.buildPlatform) '' mkdir completions diff --git a/flake.nix b/flake.nix index 7614ccb..ce7dea3 100644 --- a/flake.nix +++ b/flake.nix @@ -21,6 +21,7 @@ packages = rec { colmena = import ./default.nix { inherit pkgs; }; + # Full user manual manual = let colmena = self.packages.${system}.colmena; evalNix = import ./src/nix/eval.nix { @@ -36,7 +37,11 @@ inherit colmena deploymentOptionsMd metaOptionsMd; }; + # User manual without the CLI reference manualFast = manual.override { colmena = null; }; + + # User manual with the version treated as stable + manualForceStable = manual.override { unstable = false; }; }; defaultApp = self.apps.${system}.colmena; @@ -47,7 +52,7 @@ devShell = pkgs.mkShell { inputsFrom = [ defaultPackage ]; - nativeBuildInputs = with pkgs; [ mdbook ]; + nativeBuildInputs = with pkgs; [ mdbook python3 ]; shellHook = '' export NIX_PATH=nixpkgs=${pkgs.path} ''; diff --git a/manual/README.md b/manual/README.md index 339fb07..b81891c 100644 --- a/manual/README.md +++ b/manual/README.md @@ -7,3 +7,27 @@ You can read the rendered version [here](https://zhaofengli.github.io/colmena). The manual is rendered using [mdBook](https://github.com/rust-lang/mdBook). To build the manual, do `nix build .#manual`. You can also do `nix build .#manualFast` for a version without the CLI usage reference. + +## Marking Text for Specific Versions + +You can mark text to be only visible in the unstable version of the manual: + +``` + +You are currently reading the unstable version of the Colmena Manual. +Features described here will eventually become a part of version @apiVersion@. + +``` + +The opposite can be done with the `STABLE_{BEGIN,END}` markers: + +``` + +You are currently reading the version @apiVersion@ of the Colmena Manual. + +``` + +## Substitutions + +- `@version@` - Full version string +- `@apiVersion@` - Stable API version string (major.minor) diff --git a/manual/book.release.toml b/manual/book.stable.toml similarity index 60% rename from manual/book.release.toml rename to manual/book.stable.toml index ca09923..b63d8dc 100644 --- a/manual/book.release.toml +++ b/manual/book.stable.toml @@ -3,9 +3,13 @@ authors = ["Zhaofeng Li"] language = "en" multilingual = false src = "src" -title = "Colmena RELEASE" +title = "Colmena @apiVersion@" [output.html] -site-url = "/colmena/RELEASE/" +site-url = "/colmena/@apiVersion@/" git-repository-url = "https://github.com/zhaofengli/colmena" edit-url-template = "https://github.com/zhaofengli/colmena/edit/main/manual/{path}" + +[preprocessor.colmena] +command = "python3 preprocess.py" +renderer = [ "html", "epub" ] diff --git a/manual/book.toml b/manual/book.toml index 703e6ae..ae312f7 100644 --- a/manual/book.toml +++ b/manual/book.toml @@ -9,3 +9,7 @@ title = "Colmena (Unstable)" site-url = "/colmena/unstable/" git-repository-url = "https://github.com/zhaofengli/colmena" edit-url-template = "https://github.com/zhaofengli/colmena/edit/main/manual/{path}" + +[preprocessor.colmena] +command = "python3 preprocess.py" +renderer = [ "html", "epub" ] diff --git a/manual/default.nix b/manual/default.nix index 4af5b72..f0557be 100644 --- a/manual/default.nix +++ b/manual/default.nix @@ -2,33 +2,66 @@ let notFound = typ: "Error: No ${typ} passed to the builder"; in -{ stdenv, nix-gitignore, mdbook -, release ? null +{ lib, stdenv, nix-gitignore, mdbook, python3, writeScript , deploymentOptionsMd ? notFound "deployment options text" , metaOptionsMd ? notFound "meta options text" , colmena ? null + +, version ? null # Full version (default: detected from colmena) +, unstable ? null # Whether this build is unstable (default: detected from version) }: -stdenv.mkDerivation { - inherit colmena deploymentOptionsMd metaOptionsMd release; +let + versionComp = + if version == null then + if colmena != null then colmena.version else "unstable" + else version; - name = if release == null then "colmena-manual-dev" else "colmena-manual-${release}"; + unstableComp = + if unstable == null then versionComp == "unstable" || lib.hasInfix "-" versionComp + else unstable; + + apiVersion = builtins.concatStringsSep "." (lib.take 2 (lib.splitString "." versionComp)); + + redirectTemplate = lib.escapeShellArg '' + + + + + Redirecting + + + + Redirecting to https://zhaofengli.github.io/colmena@path@ + + + ''; + +in stdenv.mkDerivation { + inherit deploymentOptionsMd metaOptionsMd; + + pname = "colmena-manual" + (if unstableComp then "-unstable" else ""); + version = versionComp; src = nix-gitignore.gitignoreSource [] ./.; - nativeBuildInputs = [ mdbook ]; + nativeBuildInputs = [ mdbook python3 ]; + + outputs = [ "out" "redirectFarm" ]; + + COLMENA_VERSION = versionComp; + COLMENA_UNSTABLE = unstableComp; patchPhase = '' - if [ -n "$release" ]; then - find . -name '*.md' -exec sed -i "/REMOVE_FOR_RELEASE/d" {} \; - sed "s/RELEASE/$release/g" book.release.toml > book.toml + if [ -z "${toString unstableComp}" ]; then + sed "s|@apiVersion@|${apiVersion}|g" book.stable.toml > book.toml fi ''; buildPhase = '' - if [ -n "$colmena" ]; then + if [ -n "${colmena}" ]; then echo "Generating CLI help text" - $colmena/bin/colmena gen-help-markdown >> src/reference/cli.md + ${colmena}/bin/colmena gen-help-markdown >> src/reference/cli.md else echo "Error: No colmena executable passed to the builder" >> src/reference/cli.md fi @@ -37,6 +70,26 @@ stdenv.mkDerivation { echo "$metaOptionsMd" >> src/reference/meta.md mdbook build -d $out + + # Build the redirect farm + # GitHub Pages doesn't play well with directory symlinks. Default + # pages (index.html) don't work when a symlink is traversed. + + mkdir -p $redirectFarm + + subdir="/unstable" + if [ -z "${toString unstableComp}" ]; then + subdir="/${apiVersion}" + fi + + pushd $redirectFarm + (cd $out; find . -name "*.html") | while read -r page; do + strippedPage=''${page#.} + target="$subdir$strippedPage" + mkdir -p $(dirname $page) + echo ${redirectTemplate} | sed "s|@path@|$target|g" > $page + done + popd ''; installPhase = "true"; diff --git a/manual/preprocess.py b/manual/preprocess.py new file mode 100644 index 0000000..347b4c4 --- /dev/null +++ b/manual/preprocess.py @@ -0,0 +1,72 @@ +# Markdown preprocessor +# +# 1. Removes marked blocks from markdown files +# (unstable-only text from stable versions and vice versa) +# 2. Substitutes @version@ in text with the full version +# 3. Substitutes @apiVersion@ in text with major.minor +# +# Environment: +# - COLMENA_VERSION=${fullVersion} +# - COLMENA_UNSTABLE="1" or unset + +import json +import os +import pprint +import re +import sys + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +def process(book, version, unstable): + marker_to_remove = "STABLE" if unstable else "UNSTABLE" + + api_version = re.match("^[0-9]+\.[0-9]+(?=$|\.[0-9]+(.*)?$)", version) + if api_version: + api_version = api_version.group() + else: + api_version = version + + version_debug = f"{version} (apiVersion={api_version}, unstable={str(unstable)})" + + def replace_version_markers(s): + s = s.replace("@version@", version) + s = s.replace("@apiVersion@", api_version) + return s + + def process_item(item): + chapter = item["Chapter"] + + for sub_item in chapter["sub_items"]: + process_item(sub_item) + + regex = r".*\b{marker}_BEGIN\b(.|\n|\r)*?\b{marker}_END\b.*".format(marker=marker_to_remove) + + chapter["content"] = f"\n" + chapter["content"] + chapter["content"] = re.sub(regex, "", chapter["content"]) + chapter["content"] = replace_version_markers(chapter["content"]) + + #eprint(f"Processed {chapter['name']}") + + eprint(f"Version is {version_debug}") + + for section in book['sections']: + process_item(section) + +if __name__ == "__main__": + if len(sys.argv) > 1: + if sys.argv[1] == "supports": + #eprint("mdbook asked about support for", sys.argv[2]) + sys.exit(0) + + version = os.environ.get("COLMENA_VERSION", "unstable") + unstable = bool(os.environ.get("COLMENA_UNSTABLE", "")) + + if version in [ "" "unstable" ]: + unstable = True + + context, book = json.load(sys.stdin) + + process(book, version, unstable) + + print(json.dumps(book)) diff --git a/manual/src/introduction.md b/manual/src/introduction.md index 909b413..a0f8ad6 100644 --- a/manual/src/introduction.md +++ b/manual/src/introduction.md @@ -18,7 +18,14 @@ Interested? Get started [here](tutorial)! kappa ✅ 2s Activation successful -You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). + +You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). +Features described here will eventually become a part of **version @apiVersion@**. + + + +You are currently reading **version @apiVersion@** of the Colmena Manual, built against version @version@. + ## Links diff --git a/manual/src/reference/cli.md b/manual/src/reference/cli.md index 63816b1..3f6726b 100644 --- a/manual/src/reference/cli.md +++ b/manual/src/reference/cli.md @@ -1,7 +1,16 @@ # Command Line Arguments -You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). + +You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). + -The following are the auto-generated help messages that are printed when you invoke any sub-command with `--help`: +The following are the help messages that will be printed when you invoke any sub-command with `--help`: - + diff --git a/manual/src/reference/deployment.md b/manual/src/reference/deployment.md index d62f924..bbba0bb 100644 --- a/manual/src/reference/deployment.md +++ b/manual/src/reference/deployment.md @@ -1,7 +1,13 @@ # Deployment Options -You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). + +You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). + Colmena adds a set of extra options that can be used in your NixOS configurations under the `deployment` prefix. - + diff --git a/manual/src/reference/index.md b/manual/src/reference/index.md index c7c1de0..0a5872e 100644 --- a/manual/src/reference/index.md +++ b/manual/src/reference/index.md @@ -1,6 +1,8 @@ # Reference -You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). Go back to [the home page](../introduction.md) for links to other versions of the manual. + +You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). + This section contains detailed listings of options and parameters accepted by Colmena: diff --git a/manual/src/reference/meta.md b/manual/src/reference/meta.md index 11c3f87..42360ec 100644 --- a/manual/src/reference/meta.md +++ b/manual/src/reference/meta.md @@ -1,10 +1,16 @@ # Meta Options -You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). + +You are currently reading **the unstable version** of the Colmena Manual, built against the tip of [the development branch](https://github.com/zhaofengli/colmena). + The following is a list of options that can be added to the `meta` attribute set. For compatibility with NixOps, you may name it `network` instead of `meta`. However, you cannot specify both at the same time. - + diff --git a/manual/src/tutorial/flakes.md b/manual/src/tutorial/flakes.md index 8eb9884..2e88bbf 100644 --- a/manual/src/tutorial/flakes.md +++ b/manual/src/tutorial/flakes.md @@ -2,21 +2,35 @@ ## Installation + Colmena doesn't have a stable release yet. -To quickly try Colmena out, use the following command to enter an ephemeral environment with `colmena`: -```console -$ nix shell github:zhaofengli/colmena +If you are interested in trying out the bleeding-edge version of Colmena, Read [the unstable version](https://zhaofengli.github.io/colmena/unstable) of the Manual for instructions. + + + + + +To quickly try Colmena out, use the following command to enter an ephemeral environment with the latest development version of `colmena`: + +```bash +nix shell github:zhaofengli/colmena ``` -To install the latest development version to the user profile, use the following command: +To install Colmena to the user profile, use the following command: -```console -$ nix-env -if https://github.com/zhaofengli/colmena/tarball/main +```bash +nix-env -if https://github.com/zhaofengli/colmena/tarball/main ``` You can also add `github:zhaofengli/colmena` as an input in your Flake and add the `colmena` package to your `devShell`. +### Unstable Binary Cache + +A public binary cache is available at [https://colmena.cachix.org](https://colmena.cachix.org), courtesy of Cachix. +This binary cache contains unstable versions of Colmena built by [GitHub Actions](https://github.com/zhaofengli/colmena/actions). + + ## Basic Configuration Colmena reads the `colmena` output in your Flake. @@ -55,18 +69,18 @@ Here is a short example: ``` The full set of `deployment` options can be found [here](../reference/deployment.md). -For some inspiration, you can check out the short example in [the main tutorial](index.md). +You can also check out the example in [the main tutorial](index.md) for some inspiration. Now you are ready to use Colmena! To build the configuration: -```console -$ colmena build +```bash +colmena build ``` To build and deploy to all nodes: -```console -$ colmena apply +```bash +colmena apply ``` ## Next Steps diff --git a/manual/src/tutorial/index.md b/manual/src/tutorial/index.md index 5cd9492..a37f151 100644 --- a/manual/src/tutorial/index.md +++ b/manual/src/tutorial/index.md @@ -2,13 +2,27 @@ ## Installation + Colmena doesn't have a stable release yet. + +If you are interested in trying out the bleeding-edge version of Colmena, Read [the unstable version](https://zhaofengli.github.io/colmena/unstable) of the Manual for instructions. + + + + + To install the latest development version to the user profile, use the following command: -```console -$ nix-env -if https://github.com/zhaofengli/colmena/tarball/main +```bash +nix-env -if https://github.com/zhaofengli/colmena/tarball/main ``` +### Unstable Binary Cache + +A public binary cache is available at [https://colmena.cachix.org](https://colmena.cachix.org), courtesy of Cachix. +This binary cache contains unstable versions of Colmena built by [GitHub Actions](https://github.com/zhaofengli/colmena/actions). + + ## Basic Configuration *If you use Nix Flakes, follow the Flake version [here](flakes.md).* @@ -108,14 +122,14 @@ The full set of `deployment` options can be found [here](../reference/deployment Now you are ready to use Colmena! To build the configuration: -```console -$ colmena build +```bash +colmena build ``` To build and deploy to all nodes: -```console -$ colmena apply +```bash +colmena apply ``` ## Next Steps