DGNum/infrastructure
9
6
Fork 3
Code Issues 31 Pull requests 32 Projects 1 Releases Packages Wiki Activity Actions
infrastructure/patches/lon/01-npins-import.patch

625 lines
19 KiB
Diff
Raw Blame History

From 70877569a4ce8f5274c5e6208469c240a34993a0 Mon Sep 17 00:00:00 2001
From: Tom Hubrecht <tom@hubrecht.ovh>
Date: Tue, 10 Jun 2025 15:26:22 +0200
Subject: [PATCH 1/2] sources: Find default branch when none is supplied
---
rust/lon/Cargo.lock | 33 +++++++++++++++++++++++++++++++++
rust/lon/Cargo.toml | 1 +
rust/lon/src/cli.rs | 8 ++++----
rust/lon/src/git.rs | 29 +++++++++++++++++++++++++++++
rust/lon/src/init/niv.rs | 4 ++--
rust/lon/src/sources.rs | 18 +++++++++++++++---
6 files changed, 84 insertions(+), 9 deletions(-)
diff --git a/rust/lon/Cargo.lock b/rust/lon/Cargo.lock
index 62f6176..b9e7944 100644
--- a/rust/lon/Cargo.lock
+++ b/rust/lon/Cargo.lock
@@ -17,6 +17,15 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
+[[package]]
+name = "aho-corasick"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "android-tzdata"
version = "0.1.1"
@@ -847,6 +856,7 @@ dependencies = [
"expect-test",
"indoc",
"log",
+ "regex",
"reqwest",
"serde",
"serde_json",
@@ -1073,11 +1083,34 @@ dependencies = [
"getrandom 0.3.2",
]
+[[package]]
+name = "regex"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
[[package]]
name = "regex-automata"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "reqwest"
diff --git a/rust/lon/Cargo.toml b/rust/lon/Cargo.toml
index a60c24e..d7dd633 100644
--- a/rust/lon/Cargo.toml
+++ b/rust/lon/Cargo.toml
@@ -13,6 +13,7 @@ serde_json = "1.0.140"
sha2 = "0.10.9"
tempfile = "3.20.0"
reqwest = { version = "0.12", default-features = false, features = ["blocking","http2","rustls-tls","json"] }
+regex = "1.11.1"
[dev-dependencies]
expect-test = "1.5.1"
diff --git a/rust/lon/src/cli.rs b/rust/lon/src/cli.rs
index eb850d7..5806b1d 100644
--- a/rust/lon/src/cli.rs
+++ b/rust/lon/src/cli.rs
@@ -105,7 +105,7 @@ struct AddGitArgs {
/// URL to the repository
url: String,
/// Branch to track
- branch: String,
+ branch: Option<String>,
/// Revision to lock
#[arg(short, long)]
revision: Option<String>,
@@ -122,7 +122,7 @@ struct AddGitHubArgs {
/// An identifier made up of {owner}/{repo}, e.g. nixos/nixpkgs
identifier: String,
/// Branch to track
- branch: String,
+ branch: Option<String>,
/// Name of the source
///
/// If you do not supply this, the repository name is used as the source name.
@@ -283,7 +283,7 @@ fn add_git(directory: impl AsRef<Path>, args: &AddGitArgs) -> Result<()> {
let source = GitSource::new(
&args.url,
- &args.branch,
+ args.branch.as_ref(),
args.revision.as_ref(),
args.submodules,
args.frozen,
@@ -314,7 +314,7 @@ fn add_github(directory: impl AsRef<Path>, args: &AddGitHubArgs) -> Result<()> {
let source = GitHubSource::new(
owner,
repo,
- &args.branch,
+ args.branch.as_ref(),
args.revision.as_ref(),
args.frozen,
)?;
diff --git a/rust/lon/src/git.rs b/rust/lon/src/git.rs
index cb5b4df..381c337 100644
--- a/rust/lon/src/git.rs
+++ b/rust/lon/src/git.rs
@@ -5,6 +5,7 @@ use std::{
};
use anyhow::{Context, Result, bail};
+use regex::Regex;
use tempfile::TempDir;
#[derive(Clone, Debug)]
@@ -129,6 +130,34 @@ fn find_newest_revision_for_ref(url: &str, reference: &str) -> Result<Revision>
Ok(Revision(references.remove(0).revision))
}
+/// Find the default branch for a git repository
+pub fn find_default_branch(url: &str) -> Result<String> {
+ let output = Command::new("git")
+ .arg("ls-remote")
+ .args(["--symref", url, "HEAD"])
+ .output()
+ .context("Failed to execute git ls-remote. Most likely it's not on PATH")?;
+
+ if !output.status.success() {
+ bail!(
+ "Failed to find the default branch for {}\n{}",
+ url,
+ String::from_utf8_lossy(&output.stderr)
+ )
+ }
+
+ let re = Regex::new(r"ref:.*refs/heads/(?<branch>.*)\tHEAD")?;
+
+ let Some(branch) = String::from_utf8_lossy(&output.stdout)
+ .lines()
+ .find_map(|x| re.captures(x).map(|matched| matched["branch"].into()))
+ else {
+ bail!("Failed to find the default branch for {url}",)
+ };
+
+ Ok(branch)
+}
+
/// Call `git ls-remote` with the provided args.
fn ls_remote(args: &[&str]) -> Result<Vec<RemoteInfo>> {
let output = Command::new("git")
diff --git a/rust/lon/src/init/niv.rs b/rust/lon/src/init/niv.rs
index 469fdc7..8d41670 100644
--- a/rust/lon/src/init/niv.rs
+++ b/rust/lon/src/init/niv.rs
@@ -42,7 +42,7 @@ impl Convertible for LockFile {
let source = GitHubSource::new(
owner,
&package.repo,
- &package.branch,
+ Some(&package.branch),
Some(&package.rev),
false,
)?;
@@ -51,7 +51,7 @@ impl Convertible for LockFile {
} else {
let source = GitSource::new(
&package.repo,
- &package.branch,
+ Some(&package.branch),
Some(&package.rev),
false,
false,
diff --git a/rust/lon/src/sources.rs b/rust/lon/src/sources.rs
index 92d8c2b..78bdbdb 100644
--- a/rust/lon/src/sources.rs
+++ b/rust/lon/src/sources.rs
@@ -170,11 +170,16 @@ pub struct GitSource {
impl GitSource {
pub fn new(
url: &str,
- branch: &str,
+ branch: Option<&String>,
revision: Option<&String>,
submodules: bool,
frozen: bool,
) -> Result<Self> {
+ let branch = match branch {
+ Some(branch) => branch,
+ None => &git::find_default_branch(url)?,
+ };
+
let rev = match revision {
Some(rev) => rev,
None => &git::find_newest_revision(url, branch)?.to_string(),
@@ -283,13 +288,20 @@ impl GitHubSource {
pub fn new(
owner: &str,
repo: &str,
- branch: &str,
+ branch: Option<&String>,
revision: Option<&String>,
frozen: bool,
) -> Result<Self> {
+ let repo_url = &Self::git_url(owner, repo);
+
+ let branch = match branch {
+ Some(branch) => branch,
+ None => &git::find_default_branch(repo_url)?,
+ };
+
let rev = match revision {
Some(rev) => rev,
- None => &git::find_newest_revision(&Self::git_url(owner, repo), branch)?.to_string(),
+ None => &git::find_newest_revision(repo_url, branch)?.to_string(),
};
log::info!("Locked revision: {rev}");
From eee3871a246605a7ab60714bb193846160ac8e64 Mon Sep 17 00:00:00 2001
From: Tom Hubrecht <tom@hubrecht.ovh>
Date: Tue, 10 Jun 2025 17:25:52 +0200
Subject: [PATCH 2/2] cli: init from npins
We convert three types of pins: `Git`, `GitRelease` and `Channel`
---
rust/lon/src/cli.rs | 13 ++-
rust/lon/src/init.rs | 1 +
rust/lon/src/init/npins.rs | 218 +++++++++++++++++++++++++++++++++++++
rust/lon/tests/npins.json | 86 +++++++++++++++
4 files changed, 312 insertions(+), 6 deletions(-)
create mode 100644 rust/lon/src/init/npins.rs
create mode 100644 rust/lon/tests/npins.json
diff --git a/rust/lon/src/cli.rs b/rust/lon/src/cli.rs
index 5806b1d..57dcc50 100644
--- a/rust/lon/src/cli.rs
+++ b/rust/lon/src/cli.rs
@@ -11,7 +11,7 @@ use crate::{
bot::{Forge, Forgejo, GitHub, GitLab},
commit_message::CommitMessage,
git,
- init::{Convertible, niv},
+ init::{Convertible, niv, npins},
lock::Lock,
lon_nix::LonNix,
sources::{GitHubSource, GitSource, Source, Sources},
@@ -82,6 +82,7 @@ struct InitArgs {
#[derive(Clone, ValueEnum)]
enum LockFileType {
Niv,
+ Npins,
}
#[derive(Subcommand)]
@@ -261,13 +262,13 @@ fn init(directory: impl AsRef<Path>, args: &InitArgs) -> Result<()> {
bail!("No lock file type is provided");
};
- let lock_file = match lock_file_type {
- LockFileType::Niv => niv::LockFile::from_file(path)?,
- };
-
log::info!("Initializing lon.lock from {path:?}");
- let sources = lock_file.convert()?;
+ let sources = match lock_file_type {
+ LockFileType::Niv => niv::LockFile::from_file(path)?.convert()?,
+ LockFileType::Npins => npins::LockFile::from_file(path)?.convert()?,
+ };
+
sources.write(&directory)?;
Ok(())
diff --git a/rust/lon/src/init.rs b/rust/lon/src/init.rs
index ec87afa..06e63f2 100644
--- a/rust/lon/src/init.rs
+++ b/rust/lon/src/init.rs
@@ -1,4 +1,5 @@
pub mod niv;
+pub mod npins;
use anyhow::Result;
diff --git a/rust/lon/src/init/npins.rs b/rust/lon/src/init/npins.rs
new file mode 100644
index 0000000..8a38139
--- /dev/null
+++ b/rust/lon/src/init/npins.rs
@@ -0,0 +1,218 @@
+use std::{collections::BTreeMap, fs::File, io::Read, path::Path};
+
+use anyhow::{Context, Result, bail};
+use regex::Regex;
+use serde::Deserialize;
+
+use crate::{
+ init::Convertible,
+ sources::{GitHubSource, GitSource, Source, Sources},
+};
+
+#[derive(Debug, Deserialize)]
+pub struct LockFile {
+ pins: BTreeMap<String, Pin>,
+ version: u64,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(tag = "type")]
+pub enum Repository {
+ Git {
+ /// URL to the Git repository
+ url: String,
+ },
+ Forgejo {
+ server: String,
+ owner: String,
+ repo: String,
+ },
+ GitHub {
+ /// "owner/repo"
+ owner: String,
+ repo: String,
+ },
+ GitLab {
+ /// usually "owner/repo" or "group/owner/repo" (without leading or trailing slashes)
+ repo_path: String,
+ /// Of the kind <https://gitlab.example.org/>
+ ///
+ /// It must fit into the schema `<server>/<owner>/<repo>` to get a repository's URL.
+ server: String,
+ /// access token for private repositories
+ #[serde(skip_serializing_if = "Option::is_none")]
+ #[serde(default)]
+ private_token: Option<String>,
+ },
+}
+
+// HACK: We know that a Git pin has a branch associated to it and GitRelease has none,
+// but to unify the behaviour, we set them bot to `Option`s
+#[derive(Debug, Deserialize)]
+#[serde(tag = "type")]
+pub enum Pin {
+ Git {
+ repository: Repository,
+ branch: Option<String>,
+ revision: String,
+ submodules: bool,
+ #[serde(default)]
+ frozen: bool,
+ },
+ GitRelease {
+ repository: Repository,
+ branch: Option<String>,
+ revision: String,
+ submodules: bool,
+ #[serde(default)]
+ frozen: bool,
+ },
+ Channel {
+ #[serde(rename = "name")]
+ channel: String,
+ url: String,
+ #[serde(default)]
+ frozen: bool,
+ },
+}
+
+impl LockFile {
+ pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
+ let file = File::open(path.as_ref())
+ .with_context(|| format!("Failed to open {:?}", path.as_ref()))?;
+ Self::from_reader(file)
+ }
+
+ fn from_reader(rdr: impl Read) -> Result<Self> {
+ serde_json::from_reader(rdr).context("Failed to deserialize npins lock file")
+ }
+}
+
+impl Convertible for LockFile {
+ fn convert(&self) -> Result<Sources> {
+ let mut sources = Sources::default();
+
+ if self.version == 1 {
+ bail!("Unsupported npins lockfile version: {}", &self.version)
+ }
+
+ let re = Regex::new(
+ r"https://releases\.nixos\.org/.*\.(?<shortrev>[a-f0-9]+)/nixexprs\.tar\.xz",
+ )?;
+
+ for (name, pin) in &self.pins {
+ log::info!("Converting {name}...");
+
+ let source = match pin {
+ Pin::Channel {
+ channel,
+ url,
+ frozen,
+ } => {
+ let Some(matched) = re.captures(url) else {
+ bail!("Cannot extract revision from the channel url: {url}")
+ };
+
+ Source::GitHub(GitHubSource::new(
+ "NixOS",
+ "nixpkgs",
+ Some(channel),
+ Some(&matched["shortrev"].into()),
+ *frozen,
+ )?)
+ }
+ Pin::Git {
+ repository,
+ branch,
+ revision,
+ submodules,
+ frozen,
+ }
+ | Pin::GitRelease {
+ repository,
+ branch,
+ revision,
+ submodules,
+ frozen,
+ } => match repository {
+ Repository::Git { url } => Source::Git(GitSource::new(
+ url,
+ branch.as_ref(),
+ Some(revision),
+ *submodules,
+ *frozen,
+ )?),
+ Repository::GitHub { owner, repo } => {
+ if *submodules {
+ Source::Git(GitSource::new(
+ &format!("https://github.com/{owner}/{repo}"),
+ branch.as_ref(),
+ Some(revision),
+ *submodules,
+ *frozen,
+ )?)
+ } else {
+ Source::GitHub(GitHubSource::new(
+ owner,
+ repo,
+ branch.as_ref(),
+ Some(revision),
+ *frozen,
+ )?)
+ }
+ }
+ Repository::Forgejo {
+ server,
+ owner,
+ repo,
+ } => Source::Git(GitSource::new(
+ &format!("{server}/{owner}/{repo}"),
+ branch.as_ref(),
+ Some(revision),
+ *submodules,
+ *frozen,
+ )?),
+ Repository::GitLab {
+ repo_path,
+ server,
+ private_token,
+ } => {
+ if private_token.is_some() {
+ log::warn!(
+ "GitLab source {name} is configured with a PAT, which unsupported in lon"
+ );
+ }
+ Source::Git(GitSource::new(
+ &format!("{server}/{repo_path}"),
+ branch.as_ref(),
+ Some(revision),
+ *submodules,
+ *frozen,
+ )?)
+ }
+ },
+ };
+
+ sources.add(name, source);
+ }
+
+ Ok(sources)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ impl LockFile {
+ fn from_str(s: &str) -> Result<Self> {
+ serde_json::from_str(s).context("Failed to deserialize npins lock file")
+ }
+ }
+
+ #[test]
+ fn parse_npins_lock_file() -> Result<()> {
+ LockFile::from_str(include_str!("../../tests/npins.json"))?;
+ Ok(())
+ }
+}
diff --git a/rust/lon/tests/npins.json b/rust/lon/tests/npins.json
new file mode 100644
index 0000000..10ce4e2
--- /dev/null
+++ b/rust/lon/tests/npins.json
@@ -0,0 +1,86 @@
+{
+ "pins": {
+ "agenix": {
+ "type": "GitRelease",
+ "repository": {
+ "type": "GitHub",
+ "owner": "ryantm",
+ "repo": "agenix"
+ },
+ "pre_releases": false,
+ "version_upper_bound": null,
+ "release_prefix": null,
+ "submodules": false,
+ "version": "0.15.0",
+ "revision": "564595d0ad4be7277e07fa63b5a991b3c645655d",
+ "url": "https://api.github.com/repos/ryantm/agenix/tarball/refs/tags/0.15.0",
+ "hash": "sha256-ipqShkBmHKC9ft1ZAsA6aeKps32k7+XZSPwfxeHLsAU="
+ },
+ "arkheon": {
+ "type": "Git",
+ "repository": {
+ "type": "GitHub",
+ "owner": "RaitoBezarius",
+ "repo": "arkheon"
+ },
+ "branch": "main",
+ "submodules": false,
+ "revision": "3eea876b29217d01cf2ef03ea9fdd8779d28ad04",
+ "url": "https://github.com/RaitoBezarius/arkheon/archive/3eea876b29217d01cf2ef03ea9fdd8779d28ad04.tar.gz",
+ "hash": "sha256-+R6MhTXuSzNeGQiL4DQwlP5yNhmnhbf7pQWPUWgcZSM="
+ },
+ "colmena": {
+ "type": "Git",
+ "repository": {
+ "type": "Git",
+ "url": "https://git.dgnum.eu/DGNum/colmena"
+ },
+ "branch": "main",
+ "submodules": false,
+ "revision": "b5135dc8af1d7637b337cc2632990400221da577",
+ "url": null,
+ "hash": "sha256-7gg+K3PEYlN0sGPgDlmnM8zgDDIV505gNcwjFN61Qvk="
+ },
+ "nix-actions": {
+ "type": "GitRelease",
+ "repository": {
+ "type": "Git",
+ "url": "https://git.dgnum.eu/DGNum/nix-actions.git"
+ },
+ "pre_releases": false,
+ "version_upper_bound": null,
+ "release_prefix": null,
+ "submodules": false,
+ "version": "v0.5.1",
+ "revision": "06847b3256df402da0475dccb290832ec92a9f8c",
+ "url": null,
+ "hash": "sha256-2xOZdKiUfcriQFKG37vY96dgCJLndhLa7cGacq8+SA8="
+ },
+ "nixos-25.05": {
+ "type": "Channel",
+ "name": "nixos-25.05",
+ "url": "https://releases.nixos.org/nixos/25.05/nixos-25.05.803579.70c74b02eac4/nixexprs.tar.xz",
+ "hash": "sha256-0RxtgAd4gHYPFFwICal8k8hvJBOkCeTjFkh4HsqYDbE="
+ },
+ "nixos-unstable": {
+ "type": "Channel",
+ "name": "nixos-unstable",
+ "url": "https://releases.nixos.org/nixos/unstable/nixos-25.05pre797896.d89fc19e405c/nixexprs.tar.xz",
+ "hash": "sha256-bFJJ/qwB3VJ0nFuVYYHJXinT4tNJ2jhXTVT6SpYiFOM="
+ },
+ "wp4nix": {
+ "type": "Git",
+ "repository": {
+ "type": "GitLab",
+ "repo_path": "helsinki-systems/wp4nix",
+ "server": "https://git.helsinki.tools/"
+ },
+ "branch": "master",
+ "submodules": false,
+ "revision": "2fc9a0734168cab536e3129efa6397d6cd3ac89f",
+ "url": "https://git.helsinki.tools/api/v4/projects/helsinki-systems%2Fwp4nix/repository/archive.tar.gz?sha=2fc9a0734168cab536e3129efa6397d6cd3ac89f",
+ "hash": "sha256-abwqAZGsWuWqfxou8XlqedBvXsUw1/xanSgljLCJxdM="
+ }
+ },
+ "version": 6
+}
Reference in a new issue View git blame Copy permalink