chore(tools/cheddar): bump to syntect 5.0

Upgrade to syntect 5.0 and load the new kind of syntax set
serialisation with the new helper function for that purpose.

Includes other minor API fixes as well, note that the things that are
now calling `expect` previously failed internally at those points and
we're reasonably confident they don't fail in production.

This has been waiting for a long time ...

Change-Id: I8af4fef995ff64bfbe24e1f13917fa50ecb6e4ad
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7787
Reviewed-by: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI

third_party/nixpkgs/default.nix

@@ -54,10 +54,6 @@ let
   # Use `stableNixpkgs` from above.
   stableOverlay = _unstableSelf: _unstableSuper: {
     inherit (stableNixpkgs)
-      # bat syntaxes changed with syntect 5.0, but cheddar is still on 4.x
-      # TODO(tazjin): upgrade cheddar to syntect 5.0
-      bat
-
       # ntfy does not build on unstable as of 2022-08-02
       ntfy
 

tools/cheddar/Cargo.lock

@@ -184,7 +184,7 @@ dependencies = [
  "rouille",
  "serde",
  "serde_json",
- "syntect 4.6.0",
+ "syntect",
 ]
 
 [[package]]
@@ -282,7 +282,7 @@ dependencies = [
  "pest_derive",
  "regex",
  "shell-words",
- "syntect 5.0.0",
+ "syntect",
  "typed-arena",
  "unicode_categories",
  "xdg",
@@ -657,12 +657,6 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
-[[package]]
-name = "lazycell"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
-
 [[package]]
 name = "libc"
 version = "0.2.139"
@@ -1166,28 +1160,6 @@ dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "syntect"
-version = "4.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b20815bbe80ee0be06e6957450a841185fcf690fe0178f14d77a05ce2caa031"
-dependencies = [
- "bincode",
- "bitflags",
- "flate2",
- "fnv",
- "lazy_static",
- "lazycell",
- "onig",
- "plist",
- "regex-syntax",
- "serde",
- "serde_derive",
- "serde_json",
- "walkdir",
- "yaml-rust",
-]
-
 [[package]]
 name = "syntect"
 version = "5.0.0"

tools/cheddar/Cargo.toml

@@ -9,7 +9,7 @@ clap = "2.33"
 comrak = "0.15"
 lazy_static = "1.4"
 rouille = "3.6"
-syntect = "4.5.0"
+syntect = "5.0"
 serde_json = "1.0"
 regex = "1.7"
 

tools/cheddar/src/lib.rs

@@ -12,7 +12,7 @@ use std::ffi::OsStr;
 use std::io::{BufRead, Write};
 use std::path::Path;
 use std::{env, io};
-use syntect::dumps::from_binary;
+use syntect::dumps::from_uncompressed_data;
 use syntect::easy::HighlightLines;
 use syntect::highlighting::{Theme, ThemeSet};
 use syntect::parsing::{SyntaxReference, SyntaxSet};
@@ -33,7 +33,9 @@ lazy_static! {
     // Note that the syntax set is included from the path pointed to
     // by the BAT_SYNTAXES environment variable at compile time. This
     // variable is populated by Nix and points to TVL's syntax set.
-    static ref SYNTAXES: SyntaxSet = from_binary(include_bytes!(env!("BAT_SYNTAXES")));
+    static ref SYNTAXES: SyntaxSet = from_uncompressed_data(include_bytes!(env!("BAT_SYNTAXES")))
+            .expect("failed to deserialise SyntaxSet");
+
     pub static ref THEMES: ThemeSet = ThemeSet::load_defaults();
 
     // Configure Comrak's Markdown rendering with all the bells &
@@ -153,8 +155,11 @@ fn highlight_code_block(code_block: &NodeCodeBlock) -> NodeValue {
         let mut buf = BLOCK_PRE.to_string();
 
         for line in LinesWithEndings::from(&code) {
-            let regions = hl.highlight(line, &SYNTAXES);
-            append_highlighted_html_for_styled_line(&regions[..], IncludeBackground::No, &mut buf);
+            let regions = hl
+                .highlight_line(line, &SYNTAXES)
+                .expect("highlight_line failed");
+            append_highlighted_html_for_styled_line(&regions[..], IncludeBackground::No, &mut buf)
+                .expect("appending HTML failed");
         }
 
         buf.push_str("</pre>");
@@ -317,13 +322,16 @@ pub fn format_code<R: BufRead, W: Write>(
     // newlines to be efficient, and those are stripped in the lines
     // iterator.
     while should_continue(&read_result) {
-        let regions = hl.highlight(&linebuf, &SYNTAXES);
+        let regions = hl
+            .highlight_line(&linebuf, &SYNTAXES)
+            .expect("highlight_line failed");
 
         append_highlighted_html_for_styled_line(
             &regions[..],
             IncludeBackground::IfDifferent(bg),
             &mut outbuf,
-        );
+        )
+        .expect("appending highlighted HTML failed");
 
         // immediately output the current state to avoid keeping
         // things in memory

tools/cheddar/src/tests.rs

@@ -103,3 +103,8 @@ fn highlights_multiple_shortlinks() {
 fn ignores_invalid_shortlinks() {
     expect_markdown("b/abc is not a real bug", "<p>b/abc is not a real bug</p>");
 }
+
+#[test]
+fn syntax_set_loaded() {
+    assert!(SYNTAXES.syntaxes().len() > 0)
+}