diff --git a/patches/default.nix b/patches/default.nix index fd69cba..0aeb365 100644 --- a/patches/default.nix +++ b/patches/default.nix @@ -21,6 +21,10 @@ with { (local ./lix/02-fetchGit-locked.patch) ]; + lon = [ + (local ./lon/01-npins-import.patch) + ]; + "nixos-25.05" = [ # Crabfit: don't depend on all google-fonts (local ./nixpkgs/03-crabfit-karla.patch) diff --git a/patches/lon/01-npins-import.patch b/patches/lon/01-npins-import.patch new file mode 100644 index 0000000..55306a2 --- /dev/null +++ b/patches/lon/01-npins-import.patch @@ -0,0 +1,625 @@ +From 70877569a4ce8f5274c5e6208469c240a34993a0 Mon Sep 17 00:00:00 2001 +From: Tom Hubrecht +Date: Tue, 10 Jun 2025 15:26:22 +0200 +Subject: [PATCH 1/2] sources: Find default branch when none is supplied + +--- + rust/lon/Cargo.lock | 33 +++++++++++++++++++++++++++++++++ + rust/lon/Cargo.toml | 1 + + rust/lon/src/cli.rs | 8 ++++---- + rust/lon/src/git.rs | 29 +++++++++++++++++++++++++++++ + rust/lon/src/init/niv.rs | 4 ++-- + rust/lon/src/sources.rs | 18 +++++++++++++++--- + 6 files changed, 84 insertions(+), 9 deletions(-) + +diff --git a/rust/lon/Cargo.lock b/rust/lon/Cargo.lock +index 62f6176..b9e7944 100644 +--- a/rust/lon/Cargo.lock ++++ b/rust/lon/Cargo.lock +@@ -17,6 +17,15 @@ version = "2.0.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + ++[[package]] ++name = "aho-corasick" ++version = "1.1.3" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" ++dependencies = [ ++ "memchr", ++] ++ + [[package]] + name = "android-tzdata" + version = "0.1.1" +@@ -847,6 +856,7 @@ dependencies = [ + "expect-test", + "indoc", + "log", ++ "regex", + "reqwest", + "serde", + "serde_json", +@@ -1073,11 +1083,34 @@ dependencies = [ + "getrandom 0.3.2", + ] + ++[[package]] ++name = "regex" ++version = "1.11.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" ++dependencies = [ ++ "aho-corasick", ++ "memchr", ++ "regex-automata", ++ "regex-syntax", ++] ++ + [[package]] + name = "regex-automata" + version = "0.4.9" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" ++dependencies = [ ++ "aho-corasick", ++ "memchr", ++ "regex-syntax", ++] ++ ++[[package]] ++name = "regex-syntax" ++version = "0.8.5" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + + [[package]] + name = "reqwest" +diff --git a/rust/lon/Cargo.toml b/rust/lon/Cargo.toml +index a60c24e..d7dd633 100644 +--- a/rust/lon/Cargo.toml ++++ b/rust/lon/Cargo.toml +@@ -13,6 +13,7 @@ serde_json = "1.0.140" + sha2 = "0.10.9" + tempfile = "3.20.0" + reqwest = { version = "0.12", default-features = false, features = ["blocking","http2","rustls-tls","json"] } ++regex = "1.11.1" + + [dev-dependencies] + expect-test = "1.5.1" +diff --git a/rust/lon/src/cli.rs b/rust/lon/src/cli.rs +index eb850d7..5806b1d 100644 +--- a/rust/lon/src/cli.rs ++++ b/rust/lon/src/cli.rs +@@ -105,7 +105,7 @@ struct AddGitArgs { + /// URL to the repository + url: String, + /// Branch to track +- branch: String, ++ branch: Option, + /// Revision to lock + #[arg(short, long)] + revision: Option, +@@ -122,7 +122,7 @@ struct AddGitHubArgs { + /// An identifier made up of {owner}/{repo}, e.g. nixos/nixpkgs + identifier: String, + /// Branch to track +- branch: String, ++ branch: Option, + /// Name of the source + /// + /// If you do not supply this, the repository name is used as the source name. +@@ -283,7 +283,7 @@ fn add_git(directory: impl AsRef, args: &AddGitArgs) -> Result<()> { + + let source = GitSource::new( + &args.url, +- &args.branch, ++ args.branch.as_ref(), + args.revision.as_ref(), + args.submodules, + args.frozen, +@@ -314,7 +314,7 @@ fn add_github(directory: impl AsRef, args: &AddGitHubArgs) -> Result<()> { + let source = GitHubSource::new( + owner, + repo, +- &args.branch, ++ args.branch.as_ref(), + args.revision.as_ref(), + args.frozen, + )?; +diff --git a/rust/lon/src/git.rs b/rust/lon/src/git.rs +index cb5b4df..381c337 100644 +--- a/rust/lon/src/git.rs ++++ b/rust/lon/src/git.rs +@@ -5,6 +5,7 @@ use std::{ + }; + + use anyhow::{Context, Result, bail}; ++use regex::Regex; + use tempfile::TempDir; + + #[derive(Clone, Debug)] +@@ -129,6 +130,34 @@ fn find_newest_revision_for_ref(url: &str, reference: &str) -> Result + Ok(Revision(references.remove(0).revision)) + } + ++/// Find the default branch for a git repository ++pub fn find_default_branch(url: &str) -> Result { ++ let output = Command::new("git") ++ .arg("ls-remote") ++ .args(["--symref", url, "HEAD"]) ++ .output() ++ .context("Failed to execute git ls-remote. Most likely it's not on PATH")?; ++ ++ if !output.status.success() { ++ bail!( ++ "Failed to find the default branch for {}\n{}", ++ url, ++ String::from_utf8_lossy(&output.stderr) ++ ) ++ } ++ ++ let re = Regex::new(r"ref:.*refs/heads/(?.*)\tHEAD")?; ++ ++ let Some(branch) = String::from_utf8_lossy(&output.stdout) ++ .lines() ++ .find_map(|x| re.captures(x).map(|matched| matched["branch"].into())) ++ else { ++ bail!("Failed to find the default branch for {url}",) ++ }; ++ ++ Ok(branch) ++} ++ + /// Call `git ls-remote` with the provided args. + fn ls_remote(args: &[&str]) -> Result> { + let output = Command::new("git") +diff --git a/rust/lon/src/init/niv.rs b/rust/lon/src/init/niv.rs +index 469fdc7..8d41670 100644 +--- a/rust/lon/src/init/niv.rs ++++ b/rust/lon/src/init/niv.rs +@@ -42,7 +42,7 @@ impl Convertible for LockFile { + let source = GitHubSource::new( + owner, + &package.repo, +- &package.branch, ++ Some(&package.branch), + Some(&package.rev), + false, + )?; +@@ -51,7 +51,7 @@ impl Convertible for LockFile { + } else { + let source = GitSource::new( + &package.repo, +- &package.branch, ++ Some(&package.branch), + Some(&package.rev), + false, + false, +diff --git a/rust/lon/src/sources.rs b/rust/lon/src/sources.rs +index 92d8c2b..78bdbdb 100644 +--- a/rust/lon/src/sources.rs ++++ b/rust/lon/src/sources.rs +@@ -170,11 +170,16 @@ pub struct GitSource { + impl GitSource { + pub fn new( + url: &str, +- branch: &str, ++ branch: Option<&String>, + revision: Option<&String>, + submodules: bool, + frozen: bool, + ) -> Result { ++ let branch = match branch { ++ Some(branch) => branch, ++ None => &git::find_default_branch(url)?, ++ }; ++ + let rev = match revision { + Some(rev) => rev, + None => &git::find_newest_revision(url, branch)?.to_string(), +@@ -283,13 +288,20 @@ impl GitHubSource { + pub fn new( + owner: &str, + repo: &str, +- branch: &str, ++ branch: Option<&String>, + revision: Option<&String>, + frozen: bool, + ) -> Result { ++ let repo_url = &Self::git_url(owner, repo); ++ ++ let branch = match branch { ++ Some(branch) => branch, ++ None => &git::find_default_branch(repo_url)?, ++ }; ++ + let rev = match revision { + Some(rev) => rev, +- None => &git::find_newest_revision(&Self::git_url(owner, repo), branch)?.to_string(), ++ None => &git::find_newest_revision(repo_url, branch)?.to_string(), + }; + log::info!("Locked revision: {rev}"); + + +From eee3871a246605a7ab60714bb193846160ac8e64 Mon Sep 17 00:00:00 2001 +From: Tom Hubrecht +Date: Tue, 10 Jun 2025 17:25:52 +0200 +Subject: [PATCH 2/2] cli: init from npins + +We convert three types of pins: `Git`, `GitRelease` and `Channel` +--- + rust/lon/src/cli.rs | 13 ++- + rust/lon/src/init.rs | 1 + + rust/lon/src/init/npins.rs | 218 +++++++++++++++++++++++++++++++++++++ + rust/lon/tests/npins.json | 86 +++++++++++++++ + 4 files changed, 312 insertions(+), 6 deletions(-) + create mode 100644 rust/lon/src/init/npins.rs + create mode 100644 rust/lon/tests/npins.json + +diff --git a/rust/lon/src/cli.rs b/rust/lon/src/cli.rs +index 5806b1d..57dcc50 100644 +--- a/rust/lon/src/cli.rs ++++ b/rust/lon/src/cli.rs +@@ -11,7 +11,7 @@ use crate::{ + bot::{Forge, Forgejo, GitHub, GitLab}, + commit_message::CommitMessage, + git, +- init::{Convertible, niv}, ++ init::{Convertible, niv, npins}, + lock::Lock, + lon_nix::LonNix, + sources::{GitHubSource, GitSource, Source, Sources}, +@@ -82,6 +82,7 @@ struct InitArgs { + #[derive(Clone, ValueEnum)] + enum LockFileType { + Niv, ++ Npins, + } + + #[derive(Subcommand)] +@@ -261,13 +262,13 @@ fn init(directory: impl AsRef, args: &InitArgs) -> Result<()> { + bail!("No lock file type is provided"); + }; + +- let lock_file = match lock_file_type { +- LockFileType::Niv => niv::LockFile::from_file(path)?, +- }; +- + log::info!("Initializing lon.lock from {path:?}"); + +- let sources = lock_file.convert()?; ++ let sources = match lock_file_type { ++ LockFileType::Niv => niv::LockFile::from_file(path)?.convert()?, ++ LockFileType::Npins => npins::LockFile::from_file(path)?.convert()?, ++ }; ++ + sources.write(&directory)?; + + Ok(()) +diff --git a/rust/lon/src/init.rs b/rust/lon/src/init.rs +index ec87afa..06e63f2 100644 +--- a/rust/lon/src/init.rs ++++ b/rust/lon/src/init.rs +@@ -1,4 +1,5 @@ + pub mod niv; ++pub mod npins; + + use anyhow::Result; + +diff --git a/rust/lon/src/init/npins.rs b/rust/lon/src/init/npins.rs +new file mode 100644 +index 0000000..8a38139 +--- /dev/null ++++ b/rust/lon/src/init/npins.rs +@@ -0,0 +1,218 @@ ++use std::{collections::BTreeMap, fs::File, io::Read, path::Path}; ++ ++use anyhow::{Context, Result, bail}; ++use regex::Regex; ++use serde::Deserialize; ++ ++use crate::{ ++ init::Convertible, ++ sources::{GitHubSource, GitSource, Source, Sources}, ++}; ++ ++#[derive(Debug, Deserialize)] ++pub struct LockFile { ++ pins: BTreeMap, ++ version: u64, ++} ++ ++#[derive(Debug, Deserialize)] ++#[serde(tag = "type")] ++pub enum Repository { ++ Git { ++ /// URL to the Git repository ++ url: String, ++ }, ++ Forgejo { ++ server: String, ++ owner: String, ++ repo: String, ++ }, ++ GitHub { ++ /// "owner/repo" ++ owner: String, ++ repo: String, ++ }, ++ GitLab { ++ /// usually "owner/repo" or "group/owner/repo" (without leading or trailing slashes) ++ repo_path: String, ++ /// Of the kind ++ /// ++ /// It must fit into the schema `//` to get a repository's URL. ++ server: String, ++ /// access token for private repositories ++ #[serde(skip_serializing_if = "Option::is_none")] ++ #[serde(default)] ++ private_token: Option, ++ }, ++} ++ ++// HACK: We know that a Git pin has a branch associated to it and GitRelease has none, ++// but to unify the behaviour, we set them bot to `Option`s ++#[derive(Debug, Deserialize)] ++#[serde(tag = "type")] ++pub enum Pin { ++ Git { ++ repository: Repository, ++ branch: Option, ++ revision: String, ++ submodules: bool, ++ #[serde(default)] ++ frozen: bool, ++ }, ++ GitRelease { ++ repository: Repository, ++ branch: Option, ++ revision: String, ++ submodules: bool, ++ #[serde(default)] ++ frozen: bool, ++ }, ++ Channel { ++ #[serde(rename = "name")] ++ channel: String, ++ url: String, ++ #[serde(default)] ++ frozen: bool, ++ }, ++} ++ ++impl LockFile { ++ pub fn from_file(path: impl AsRef) -> Result { ++ let file = File::open(path.as_ref()) ++ .with_context(|| format!("Failed to open {:?}", path.as_ref()))?; ++ Self::from_reader(file) ++ } ++ ++ fn from_reader(rdr: impl Read) -> Result { ++ serde_json::from_reader(rdr).context("Failed to deserialize npins lock file") ++ } ++} ++ ++impl Convertible for LockFile { ++ fn convert(&self) -> Result { ++ let mut sources = Sources::default(); ++ ++ if self.version == 1 { ++ bail!("Unsupported npins lockfile version: {}", &self.version) ++ } ++ ++ let re = Regex::new( ++ r"https://releases\.nixos\.org/.*\.(?[a-f0-9]+)/nixexprs\.tar\.xz", ++ )?; ++ ++ for (name, pin) in &self.pins { ++ log::info!("Converting {name}..."); ++ ++ let source = match pin { ++ Pin::Channel { ++ channel, ++ url, ++ frozen, ++ } => { ++ let Some(matched) = re.captures(url) else { ++ bail!("Cannot extract revision from the channel url: {url}") ++ }; ++ ++ Source::GitHub(GitHubSource::new( ++ "NixOS", ++ "nixpkgs", ++ Some(channel), ++ Some(&matched["shortrev"].into()), ++ *frozen, ++ )?) ++ } ++ Pin::Git { ++ repository, ++ branch, ++ revision, ++ submodules, ++ frozen, ++ } ++ | Pin::GitRelease { ++ repository, ++ branch, ++ revision, ++ submodules, ++ frozen, ++ } => match repository { ++ Repository::Git { url } => Source::Git(GitSource::new( ++ url, ++ branch.as_ref(), ++ Some(revision), ++ *submodules, ++ *frozen, ++ )?), ++ Repository::GitHub { owner, repo } => { ++ if *submodules { ++ Source::Git(GitSource::new( ++ &format!("https://github.com/{owner}/{repo}"), ++ branch.as_ref(), ++ Some(revision), ++ *submodules, ++ *frozen, ++ )?) ++ } else { ++ Source::GitHub(GitHubSource::new( ++ owner, ++ repo, ++ branch.as_ref(), ++ Some(revision), ++ *frozen, ++ )?) ++ } ++ } ++ Repository::Forgejo { ++ server, ++ owner, ++ repo, ++ } => Source::Git(GitSource::new( ++ &format!("{server}/{owner}/{repo}"), ++ branch.as_ref(), ++ Some(revision), ++ *submodules, ++ *frozen, ++ )?), ++ Repository::GitLab { ++ repo_path, ++ server, ++ private_token, ++ } => { ++ if private_token.is_some() { ++ log::warn!( ++ "GitLab source {name} is configured with a PAT, which unsupported in lon" ++ ); ++ } ++ Source::Git(GitSource::new( ++ &format!("{server}/{repo_path}"), ++ branch.as_ref(), ++ Some(revision), ++ *submodules, ++ *frozen, ++ )?) ++ } ++ }, ++ }; ++ ++ sources.add(name, source); ++ } ++ ++ Ok(sources) ++ } ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ ++ impl LockFile { ++ fn from_str(s: &str) -> Result { ++ serde_json::from_str(s).context("Failed to deserialize npins lock file") ++ } ++ } ++ ++ #[test] ++ fn parse_npins_lock_file() -> Result<()> { ++ LockFile::from_str(include_str!("../../tests/npins.json"))?; ++ Ok(()) ++ } ++} +diff --git a/rust/lon/tests/npins.json b/rust/lon/tests/npins.json +new file mode 100644 +index 0000000..10ce4e2 +--- /dev/null ++++ b/rust/lon/tests/npins.json +@@ -0,0 +1,86 @@ ++{ ++ "pins": { ++ "agenix": { ++ "type": "GitRelease", ++ "repository": { ++ "type": "GitHub", ++ "owner": "ryantm", ++ "repo": "agenix" ++ }, ++ "pre_releases": false, ++ "version_upper_bound": null, ++ "release_prefix": null, ++ "submodules": false, ++ "version": "0.15.0", ++ "revision": "564595d0ad4be7277e07fa63b5a991b3c645655d", ++ "url": "https://api.github.com/repos/ryantm/agenix/tarball/refs/tags/0.15.0", ++ "hash": "sha256-ipqShkBmHKC9ft1ZAsA6aeKps32k7+XZSPwfxeHLsAU=" ++ }, ++ "arkheon": { ++ "type": "Git", ++ "repository": { ++ "type": "GitHub", ++ "owner": "RaitoBezarius", ++ "repo": "arkheon" ++ }, ++ "branch": "main", ++ "submodules": false, ++ "revision": "3eea876b29217d01cf2ef03ea9fdd8779d28ad04", ++ "url": "https://github.com/RaitoBezarius/arkheon/archive/3eea876b29217d01cf2ef03ea9fdd8779d28ad04.tar.gz", ++ "hash": "sha256-+R6MhTXuSzNeGQiL4DQwlP5yNhmnhbf7pQWPUWgcZSM=" ++ }, ++ "colmena": { ++ "type": "Git", ++ "repository": { ++ "type": "Git", ++ "url": "https://git.dgnum.eu/DGNum/colmena" ++ }, ++ "branch": "main", ++ "submodules": false, ++ "revision": "b5135dc8af1d7637b337cc2632990400221da577", ++ "url": null, ++ "hash": "sha256-7gg+K3PEYlN0sGPgDlmnM8zgDDIV505gNcwjFN61Qvk=" ++ }, ++ "nix-actions": { ++ "type": "GitRelease", ++ "repository": { ++ "type": "Git", ++ "url": "https://git.dgnum.eu/DGNum/nix-actions.git" ++ }, ++ "pre_releases": false, ++ "version_upper_bound": null, ++ "release_prefix": null, ++ "submodules": false, ++ "version": "v0.5.1", ++ "revision": "06847b3256df402da0475dccb290832ec92a9f8c", ++ "url": null, ++ "hash": "sha256-2xOZdKiUfcriQFKG37vY96dgCJLndhLa7cGacq8+SA8=" ++ }, ++ "nixos-25.05": { ++ "type": "Channel", ++ "name": "nixos-25.05", ++ "url": "https://releases.nixos.org/nixos/25.05/nixos-25.05.803579.70c74b02eac4/nixexprs.tar.xz", ++ "hash": "sha256-0RxtgAd4gHYPFFwICal8k8hvJBOkCeTjFkh4HsqYDbE=" ++ }, ++ "nixos-unstable": { ++ "type": "Channel", ++ "name": "nixos-unstable", ++ "url": "https://releases.nixos.org/nixos/unstable/nixos-25.05pre797896.d89fc19e405c/nixexprs.tar.xz", ++ "hash": "sha256-bFJJ/qwB3VJ0nFuVYYHJXinT4tNJ2jhXTVT6SpYiFOM=" ++ }, ++ "wp4nix": { ++ "type": "Git", ++ "repository": { ++ "type": "GitLab", ++ "repo_path": "helsinki-systems/wp4nix", ++ "server": "https://git.helsinki.tools/" ++ }, ++ "branch": "master", ++ "submodules": false, ++ "revision": "2fc9a0734168cab536e3129efa6397d6cd3ac89f", ++ "url": "https://git.helsinki.tools/api/v4/projects/helsinki-systems%2Fwp4nix/repository/archive.tar.gz?sha=2fc9a0734168cab536e3129efa6397d6cd3ac89f", ++ "hash": "sha256-abwqAZGsWuWqfxou8XlqedBvXsUw1/xanSgljLCJxdM=" ++ } ++ }, ++ "version": 6 ++} diff --git a/patches/lon/01-npins-import.patch.license b/patches/lon/01-npins-import.patch.license new file mode 100644 index 0000000..12f3979 --- /dev/null +++ b/patches/lon/01-npins-import.patch.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2025 Tom Hubrecht + +SPDX-License-Identifier: MIT