From 70877569a4ce8f5274c5e6208469c240a34993a0 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 10 Jun 2025 15:26:22 +0200 Subject: [PATCH 1/2] sources: Find default branch when none is supplied --- rust/lon/Cargo.lock | 33 +++++++++++++++++++++++++++++++++ rust/lon/Cargo.toml | 1 + rust/lon/src/cli.rs | 8 ++++---- rust/lon/src/git.rs | 29 +++++++++++++++++++++++++++++ rust/lon/src/init/niv.rs | 4 ++-- rust/lon/src/sources.rs | 18 +++++++++++++++--- 6 files changed, 84 insertions(+), 9 deletions(-) diff --git a/rust/lon/Cargo.lock b/rust/lon/Cargo.lock index 62f6176..b9e7944 100644 --- a/rust/lon/Cargo.lock +++ b/rust/lon/Cargo.lock @@ -17,6 +17,15 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -847,6 +856,7 @@ dependencies = [ "expect-test", "indoc", "log", + "regex", "reqwest", "serde", "serde_json", @@ -1073,11 +1083,34 @@ dependencies = [ "getrandom 0.3.2", ] +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" diff --git a/rust/lon/Cargo.toml b/rust/lon/Cargo.toml index a60c24e..d7dd633 100644 --- a/rust/lon/Cargo.toml +++ b/rust/lon/Cargo.toml @@ -13,6 +13,7 @@ serde_json = "1.0.140" sha2 = "0.10.9" tempfile = "3.20.0" reqwest = { version = "0.12", default-features = false, features = ["blocking","http2","rustls-tls","json"] } +regex = "1.11.1" [dev-dependencies] expect-test = "1.5.1" diff --git a/rust/lon/src/cli.rs b/rust/lon/src/cli.rs index eb850d7..5806b1d 100644 --- a/rust/lon/src/cli.rs +++ b/rust/lon/src/cli.rs @@ -105,7 +105,7 @@ struct AddGitArgs { /// URL to the repository url: String, /// Branch to track - branch: String, + branch: Option, /// Revision to lock #[arg(short, long)] revision: Option, @@ -122,7 +122,7 @@ struct AddGitHubArgs { /// An identifier made up of {owner}/{repo}, e.g. nixos/nixpkgs identifier: String, /// Branch to track - branch: String, + branch: Option, /// Name of the source /// /// If you do not supply this, the repository name is used as the source name. @@ -283,7 +283,7 @@ fn add_git(directory: impl AsRef, args: &AddGitArgs) -> Result<()> { let source = GitSource::new( &args.url, - &args.branch, + args.branch.as_ref(), args.revision.as_ref(), args.submodules, args.frozen, @@ -314,7 +314,7 @@ fn add_github(directory: impl AsRef, args: &AddGitHubArgs) -> Result<()> { let source = GitHubSource::new( owner, repo, - &args.branch, + args.branch.as_ref(), args.revision.as_ref(), args.frozen, )?; diff --git a/rust/lon/src/git.rs b/rust/lon/src/git.rs index cb5b4df..381c337 100644 --- a/rust/lon/src/git.rs +++ b/rust/lon/src/git.rs @@ -5,6 +5,7 @@ use std::{ }; use anyhow::{Context, Result, bail}; +use regex::Regex; use tempfile::TempDir; #[derive(Clone, Debug)] @@ -129,6 +130,34 @@ fn find_newest_revision_for_ref(url: &str, reference: &str) -> Result Ok(Revision(references.remove(0).revision)) } +/// Find the default branch for a git repository +pub fn find_default_branch(url: &str) -> Result { + let output = Command::new("git") + .arg("ls-remote") + .args(["--symref", url, "HEAD"]) + .output() + .context("Failed to execute git ls-remote. Most likely it's not on PATH")?; + + if !output.status.success() { + bail!( + "Failed to find the default branch for {}\n{}", + url, + String::from_utf8_lossy(&output.stderr) + ) + } + + let re = Regex::new(r"ref:.*refs/heads/(?.*)\tHEAD")?; + + let Some(branch) = String::from_utf8_lossy(&output.stdout) + .lines() + .find_map(|x| re.captures(x).map(|matched| matched["branch"].into())) + else { + bail!("Failed to find the default branch for {url}",) + }; + + Ok(branch) +} + /// Call `git ls-remote` with the provided args. fn ls_remote(args: &[&str]) -> Result> { let output = Command::new("git") diff --git a/rust/lon/src/init/niv.rs b/rust/lon/src/init/niv.rs index 469fdc7..8d41670 100644 --- a/rust/lon/src/init/niv.rs +++ b/rust/lon/src/init/niv.rs @@ -42,7 +42,7 @@ impl Convertible for LockFile { let source = GitHubSource::new( owner, &package.repo, - &package.branch, + Some(&package.branch), Some(&package.rev), false, )?; @@ -51,7 +51,7 @@ impl Convertible for LockFile { } else { let source = GitSource::new( &package.repo, - &package.branch, + Some(&package.branch), Some(&package.rev), false, false, diff --git a/rust/lon/src/sources.rs b/rust/lon/src/sources.rs index 92d8c2b..78bdbdb 100644 --- a/rust/lon/src/sources.rs +++ b/rust/lon/src/sources.rs @@ -170,11 +170,16 @@ pub struct GitSource { impl GitSource { pub fn new( url: &str, - branch: &str, + branch: Option<&String>, revision: Option<&String>, submodules: bool, frozen: bool, ) -> Result { + let branch = match branch { + Some(branch) => branch, + None => &git::find_default_branch(url)?, + }; + let rev = match revision { Some(rev) => rev, None => &git::find_newest_revision(url, branch)?.to_string(), @@ -283,13 +288,20 @@ impl GitHubSource { pub fn new( owner: &str, repo: &str, - branch: &str, + branch: Option<&String>, revision: Option<&String>, frozen: bool, ) -> Result { + let repo_url = &Self::git_url(owner, repo); + + let branch = match branch { + Some(branch) => branch, + None => &git::find_default_branch(repo_url)?, + }; + let rev = match revision { Some(rev) => rev, - None => &git::find_newest_revision(&Self::git_url(owner, repo), branch)?.to_string(), + None => &git::find_newest_revision(repo_url, branch)?.to_string(), }; log::info!("Locked revision: {rev}"); From eee3871a246605a7ab60714bb193846160ac8e64 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Tue, 10 Jun 2025 17:25:52 +0200 Subject: [PATCH 2/2] cli: init from npins We convert three types of pins: `Git`, `GitRelease` and `Channel` --- rust/lon/src/cli.rs | 13 ++- rust/lon/src/init.rs | 1 + rust/lon/src/init/npins.rs | 218 +++++++++++++++++++++++++++++++++++++ rust/lon/tests/npins.json | 86 +++++++++++++++ 4 files changed, 312 insertions(+), 6 deletions(-) create mode 100644 rust/lon/src/init/npins.rs create mode 100644 rust/lon/tests/npins.json diff --git a/rust/lon/src/cli.rs b/rust/lon/src/cli.rs index 5806b1d..57dcc50 100644 --- a/rust/lon/src/cli.rs +++ b/rust/lon/src/cli.rs @@ -11,7 +11,7 @@ use crate::{ bot::{Forge, Forgejo, GitHub, GitLab}, commit_message::CommitMessage, git, - init::{Convertible, niv}, + init::{Convertible, niv, npins}, lock::Lock, lon_nix::LonNix, sources::{GitHubSource, GitSource, Source, Sources}, @@ -82,6 +82,7 @@ struct InitArgs { #[derive(Clone, ValueEnum)] enum LockFileType { Niv, + Npins, } #[derive(Subcommand)] @@ -261,13 +262,13 @@ fn init(directory: impl AsRef, args: &InitArgs) -> Result<()> { bail!("No lock file type is provided"); }; - let lock_file = match lock_file_type { - LockFileType::Niv => niv::LockFile::from_file(path)?, - }; - log::info!("Initializing lon.lock from {path:?}"); - let sources = lock_file.convert()?; + let sources = match lock_file_type { + LockFileType::Niv => niv::LockFile::from_file(path)?.convert()?, + LockFileType::Npins => npins::LockFile::from_file(path)?.convert()?, + }; + sources.write(&directory)?; Ok(()) diff --git a/rust/lon/src/init.rs b/rust/lon/src/init.rs index ec87afa..06e63f2 100644 --- a/rust/lon/src/init.rs +++ b/rust/lon/src/init.rs @@ -1,4 +1,5 @@ pub mod niv; +pub mod npins; use anyhow::Result; diff --git a/rust/lon/src/init/npins.rs b/rust/lon/src/init/npins.rs new file mode 100644 index 0000000..8a38139 --- /dev/null +++ b/rust/lon/src/init/npins.rs @@ -0,0 +1,218 @@ +use std::{collections::BTreeMap, fs::File, io::Read, path::Path}; + +use anyhow::{Context, Result, bail}; +use regex::Regex; +use serde::Deserialize; + +use crate::{ + init::Convertible, + sources::{GitHubSource, GitSource, Source, Sources}, +}; + +#[derive(Debug, Deserialize)] +pub struct LockFile { + pins: BTreeMap, + version: u64, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "type")] +pub enum Repository { + Git { + /// URL to the Git repository + url: String, + }, + Forgejo { + server: String, + owner: String, + repo: String, + }, + GitHub { + /// "owner/repo" + owner: String, + repo: String, + }, + GitLab { + /// usually "owner/repo" or "group/owner/repo" (without leading or trailing slashes) + repo_path: String, + /// Of the kind + /// + /// It must fit into the schema `//` to get a repository's URL. + server: String, + /// access token for private repositories + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + private_token: Option, + }, +} + +// HACK: We know that a Git pin has a branch associated to it and GitRelease has none, +// but to unify the behaviour, we set them bot to `Option`s +#[derive(Debug, Deserialize)] +#[serde(tag = "type")] +pub enum Pin { + Git { + repository: Repository, + branch: Option, + revision: String, + submodules: bool, + #[serde(default)] + frozen: bool, + }, + GitRelease { + repository: Repository, + branch: Option, + revision: String, + submodules: bool, + #[serde(default)] + frozen: bool, + }, + Channel { + #[serde(rename = "name")] + channel: String, + url: String, + #[serde(default)] + frozen: bool, + }, +} + +impl LockFile { + pub fn from_file(path: impl AsRef) -> Result { + let file = File::open(path.as_ref()) + .with_context(|| format!("Failed to open {:?}", path.as_ref()))?; + Self::from_reader(file) + } + + fn from_reader(rdr: impl Read) -> Result { + serde_json::from_reader(rdr).context("Failed to deserialize npins lock file") + } +} + +impl Convertible for LockFile { + fn convert(&self) -> Result { + let mut sources = Sources::default(); + + if self.version == 1 { + bail!("Unsupported npins lockfile version: {}", &self.version) + } + + let re = Regex::new( + r"https://releases\.nixos\.org/.*\.(?[a-f0-9]+)/nixexprs\.tar\.xz", + )?; + + for (name, pin) in &self.pins { + log::info!("Converting {name}..."); + + let source = match pin { + Pin::Channel { + channel, + url, + frozen, + } => { + let Some(matched) = re.captures(url) else { + bail!("Cannot extract revision from the channel url: {url}") + }; + + Source::GitHub(GitHubSource::new( + "NixOS", + "nixpkgs", + Some(channel), + Some(&matched["shortrev"].into()), + *frozen, + )?) + } + Pin::Git { + repository, + branch, + revision, + submodules, + frozen, + } + | Pin::GitRelease { + repository, + branch, + revision, + submodules, + frozen, + } => match repository { + Repository::Git { url } => Source::Git(GitSource::new( + url, + branch.as_ref(), + Some(revision), + *submodules, + *frozen, + )?), + Repository::GitHub { owner, repo } => { + if *submodules { + Source::Git(GitSource::new( + &format!("https://github.com/{owner}/{repo}"), + branch.as_ref(), + Some(revision), + *submodules, + *frozen, + )?) + } else { + Source::GitHub(GitHubSource::new( + owner, + repo, + branch.as_ref(), + Some(revision), + *frozen, + )?) + } + } + Repository::Forgejo { + server, + owner, + repo, + } => Source::Git(GitSource::new( + &format!("{server}/{owner}/{repo}"), + branch.as_ref(), + Some(revision), + *submodules, + *frozen, + )?), + Repository::GitLab { + repo_path, + server, + private_token, + } => { + if private_token.is_some() { + log::warn!( + "GitLab source {name} is configured with a PAT, which unsupported in lon" + ); + } + Source::Git(GitSource::new( + &format!("{server}/{repo_path}"), + branch.as_ref(), + Some(revision), + *submodules, + *frozen, + )?) + } + }, + }; + + sources.add(name, source); + } + + Ok(sources) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + impl LockFile { + fn from_str(s: &str) -> Result { + serde_json::from_str(s).context("Failed to deserialize npins lock file") + } + } + + #[test] + fn parse_npins_lock_file() -> Result<()> { + LockFile::from_str(include_str!("../../tests/npins.json"))?; + Ok(()) + } +} diff --git a/rust/lon/tests/npins.json b/rust/lon/tests/npins.json new file mode 100644 index 0000000..10ce4e2 --- /dev/null +++ b/rust/lon/tests/npins.json @@ -0,0 +1,86 @@ +{ + "pins": { + "agenix": { + "type": "GitRelease", + "repository": { + "type": "GitHub", + "owner": "ryantm", + "repo": "agenix" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, + "submodules": false, + "version": "0.15.0", + "revision": "564595d0ad4be7277e07fa63b5a991b3c645655d", + "url": "https://api.github.com/repos/ryantm/agenix/tarball/refs/tags/0.15.0", + "hash": "sha256-ipqShkBmHKC9ft1ZAsA6aeKps32k7+XZSPwfxeHLsAU=" + }, + "arkheon": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "RaitoBezarius", + "repo": "arkheon" + }, + "branch": "main", + "submodules": false, + "revision": "3eea876b29217d01cf2ef03ea9fdd8779d28ad04", + "url": "https://github.com/RaitoBezarius/arkheon/archive/3eea876b29217d01cf2ef03ea9fdd8779d28ad04.tar.gz", + "hash": "sha256-+R6MhTXuSzNeGQiL4DQwlP5yNhmnhbf7pQWPUWgcZSM=" + }, + "colmena": { + "type": "Git", + "repository": { + "type": "Git", + "url": "https://git.dgnum.eu/DGNum/colmena" + }, + "branch": "main", + "submodules": false, + "revision": "b5135dc8af1d7637b337cc2632990400221da577", + "url": null, + "hash": "sha256-7gg+K3PEYlN0sGPgDlmnM8zgDDIV505gNcwjFN61Qvk=" + }, + "nix-actions": { + "type": "GitRelease", + "repository": { + "type": "Git", + "url": "https://git.dgnum.eu/DGNum/nix-actions.git" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, + "submodules": false, + "version": "v0.5.1", + "revision": "06847b3256df402da0475dccb290832ec92a9f8c", + "url": null, + "hash": "sha256-2xOZdKiUfcriQFKG37vY96dgCJLndhLa7cGacq8+SA8=" + }, + "nixos-25.05": { + "type": "Channel", + "name": "nixos-25.05", + "url": "https://releases.nixos.org/nixos/25.05/nixos-25.05.803579.70c74b02eac4/nixexprs.tar.xz", + "hash": "sha256-0RxtgAd4gHYPFFwICal8k8hvJBOkCeTjFkh4HsqYDbE=" + }, + "nixos-unstable": { + "type": "Channel", + "name": "nixos-unstable", + "url": "https://releases.nixos.org/nixos/unstable/nixos-25.05pre797896.d89fc19e405c/nixexprs.tar.xz", + "hash": "sha256-bFJJ/qwB3VJ0nFuVYYHJXinT4tNJ2jhXTVT6SpYiFOM=" + }, + "wp4nix": { + "type": "Git", + "repository": { + "type": "GitLab", + "repo_path": "helsinki-systems/wp4nix", + "server": "https://git.helsinki.tools/" + }, + "branch": "master", + "submodules": false, + "revision": "2fc9a0734168cab536e3129efa6397d6cd3ac89f", + "url": "https://git.helsinki.tools/api/v4/projects/helsinki-systems%2Fwp4nix/repository/archive.tar.gz?sha=2fc9a0734168cab536e3129efa6397d6cd3ac89f", + "hash": "sha256-abwqAZGsWuWqfxou8XlqedBvXsUw1/xanSgljLCJxdM=" + } + }, + "version": 6 +}