Run rustfmt

This commit is contained in:
Zhaofeng Li 2022-07-29 22:13:09 -07:00
parent e82993a83e
commit 62a3d1e6f8
37 changed files with 1241 additions and 847 deletions

View file

@ -74,7 +74,7 @@
packages = with pkgs; [ packages = with pkgs; [
bashInteractive bashInteractive
editorconfig-checker editorconfig-checker
clippy rust-analyzer cargo-outdated clippy rust-analyzer cargo-outdated rustfmt
python3 python3Packages.flake8 python3 python3Packages.flake8
]; ];
}; };

View file

@ -2,7 +2,7 @@
use std::env; use std::env;
use clap::{Command as ClapCommand, Arg, ArgMatches, ColorChoice}; use clap::{Arg, ArgMatches, ColorChoice, Command as ClapCommand};
use clap_complete::Shell; use clap_complete::Shell;
use const_format::concatcp; use const_format::concatcp;
use env_logger::fmt::WriteStyle; use env_logger::fmt::WriteStyle;
@ -18,7 +18,13 @@ const MANUAL_URL_BASE: &str = "https://colmena.cli.rs";
/// We maintain CLI and Nix API stability for each minor version. /// We maintain CLI and Nix API stability for each minor version.
/// This ensures that the user always sees accurate documentations, and we can /// This ensures that the user always sees accurate documentations, and we can
/// easily perform updates to the manual after a release. /// easily perform updates to the manual after a release.
const MANUAL_URL: &str = concatcp!(MANUAL_URL_BASE, "/", env!("CARGO_PKG_VERSION_MAJOR"), ".", env!("CARGO_PKG_VERSION_MINOR")); const MANUAL_URL: &str = concatcp!(
MANUAL_URL_BASE,
"/",
env!("CARGO_PKG_VERSION_MAJOR"),
".",
env!("CARGO_PKG_VERSION_MINOR")
);
/// The note shown when the user is using a pre-release version. /// The note shown when the user is using a pre-release version.
/// ///
@ -29,12 +35,15 @@ const MANUAL_DISCREPANCY_NOTE: &str = "Note: You are using a pre-release version
lazy_static! { lazy_static! {
static ref LONG_ABOUT: String = { static ref LONG_ABOUT: String = {
let mut message = format!(r#"NixOS deployment tool let mut message = format!(
r#"NixOS deployment tool
Colmena helps you deploy to multiple hosts running NixOS. Colmena helps you deploy to multiple hosts running NixOS.
For more details, read the manual at <{}>. For more details, read the manual at <{}>.
"#, MANUAL_URL); "#,
MANUAL_URL
);
if !env!("CARGO_PKG_VERSION_PRE").is_empty() { if !env!("CARGO_PKG_VERSION_PRE").is_empty() {
message += MANUAL_DISCREPANCY_NOTE; message += MANUAL_DISCREPANCY_NOTE;
@ -42,12 +51,14 @@ For more details, read the manual at <{}>.
message message
}; };
static ref CONFIG_HELP: String = { static ref CONFIG_HELP: String = {
format!(r#"If this argument is not specified, Colmena will search upwards from the current working directory for a file named "flake.nix" or "hive.nix". This behavior is disabled if --config/-f is given explicitly. format!(
r#"If this argument is not specified, Colmena will search upwards from the current working directory for a file named "flake.nix" or "hive.nix". This behavior is disabled if --config/-f is given explicitly.
For a sample configuration, check the manual at <{}>. For a sample configuration, check the manual at <{}>.
"#, MANUAL_URL) "#,
MANUAL_URL
)
}; };
} }
@ -68,19 +79,15 @@ macro_rules! register_command {
macro_rules! handle_command { macro_rules! handle_command {
($module:ident, $matches:ident) => { ($module:ident, $matches:ident) => {
if let Some(sub_matches) = $matches.subcommand_matches(stringify!($module)) { if let Some(sub_matches) = $matches.subcommand_matches(stringify!($module)) {
crate::troubleshooter::run_wrapped( crate::troubleshooter::run_wrapped(&$matches, &sub_matches, command::$module::run)
&$matches, &sub_matches, .await;
command::$module::run,
).await;
return; return;
} }
}; };
($name:expr, $module:ident, $matches:ident) => { ($name:expr, $module:ident, $matches:ident) => {
if let Some(sub_matches) = $matches.subcommand_matches($name) { if let Some(sub_matches) = $matches.subcommand_matches($name) {
crate::troubleshooter::run_wrapped( crate::troubleshooter::run_wrapped(&$matches, &sub_matches, command::$module::run)
&$matches, &sub_matches, .await;
command::$module::run,
).await;
return; return;
} }
}; };
@ -131,14 +138,18 @@ It's also possible to specify the preference using environment variables. See <h
.global(true)); .global(true));
if include_internal { if include_internal {
app = app.subcommand(ClapCommand::new("gen-completions") app = app.subcommand(
.about("Generate shell auto-completion files (Internal)") ClapCommand::new("gen-completions")
.hide(true) .about("Generate shell auto-completion files (Internal)")
.arg(Arg::new("shell") .hide(true)
.index(1) .arg(
.possible_values(Shell::possible_values()) Arg::new("shell")
.required(true) .index(1)
.takes_value(true))); .possible_values(Shell::possible_values())
.required(true)
.takes_value(true),
),
);
// deprecated alias // deprecated alias
app = app.subcommand(command::eval::deprecated_alias()); app = app.subcommand(command::eval::deprecated_alias());

View file

@ -1,19 +1,14 @@
use std::env; use std::env;
use std::path::PathBuf; use std::path::PathBuf;
use clap::{Arg, Command as ClapCommand, ArgMatches}; use clap::{Arg, ArgMatches, Command as ClapCommand};
use crate::error::ColmenaError; use crate::error::ColmenaError;
use crate::nix::deployment::{ use crate::nix::deployment::{
Deployment, Deployment, EvaluationNodeLimit, Evaluator, Goal, Options, ParallelismLimit,
Goal,
Options,
EvaluationNodeLimit,
Evaluator,
ParallelismLimit,
}; };
use crate::progress::SimpleProgressOutput;
use crate::nix::NodeFilter; use crate::nix::NodeFilter;
use crate::progress::SimpleProgressOutput;
use crate::util; use crate::util;
pub fn register_deploy_args(command: ClapCommand) -> ClapCommand { pub fn register_deploy_args(command: ClapCommand) -> ClapCommand {
@ -145,27 +140,26 @@ pub fn subcommand() -> ClapCommand<'static> {
pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(), ColmenaError> { pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(), ColmenaError> {
let hive = util::hive_from_args(local_args).await?; let hive = util::hive_from_args(local_args).await?;
let ssh_config = env::var("SSH_CONFIG_FILE") let ssh_config = env::var("SSH_CONFIG_FILE").ok().map(PathBuf::from);
.ok().map(PathBuf::from);
let goal_arg = local_args.value_of("goal").unwrap(); let goal_arg = local_args.value_of("goal").unwrap();
let goal = Goal::from_str(goal_arg).unwrap(); let goal = Goal::from_str(goal_arg).unwrap();
let filter = local_args.value_of("on") let filter = local_args.value_of("on").map(NodeFilter::new).transpose()?;
.map(NodeFilter::new)
.transpose()?;
if !filter.is_some() && goal != Goal::Build { if !filter.is_some() && goal != Goal::Build {
// User did not specify node, we should check meta and see rules // User did not specify node, we should check meta and see rules
let meta = hive.get_meta_config().await?; let meta = hive.get_meta_config().await?;
if !meta.allow_apply_all { if !meta.allow_apply_all {
log::error!("No node filter is specified and meta.allowApplyAll is set to false."); log::error!("No node filter is specified and meta.allowApplyAll is set to false.");
log::error!("Hint: Filter the nodes with --on."); log::error!("Hint: Filter the nodes with --on.");
quit::with_code(1); quit::with_code(1);
} }
} }
let targets = hive.select_nodes(filter, ssh_config, goal.requires_target_host()).await?; let targets = hive
.select_nodes(filter, ssh_config, goal.requires_target_host())
.await?;
let n_targets = targets.len(); let n_targets = targets.len();
let verbose = local_args.is_present("verbose") || goal == Goal::DryActivate; let verbose = local_args.is_present("verbose") || goal == Goal::DryActivate;
@ -181,7 +175,9 @@ pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(
options.set_gzip(!local_args.is_present("no-gzip")); options.set_gzip(!local_args.is_present("no-gzip"));
options.set_upload_keys(!local_args.is_present("no-keys")); options.set_upload_keys(!local_args.is_present("no-keys"));
options.set_reboot(local_args.is_present("reboot")); options.set_reboot(local_args.is_present("reboot"));
options.set_force_replace_unknown_profiles(local_args.is_present("force-replace-unknown-profiles")); options.set_force_replace_unknown_profiles(
local_args.is_present("force-replace-unknown-profiles"),
);
options.set_evaluator(local_args.value_of_t("evaluator").unwrap()); options.set_evaluator(local_args.value_of_t("evaluator").unwrap());
if local_args.is_present("keep-result") { if local_args.is_present("keep-result") {
@ -207,7 +203,11 @@ pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(
let parallelism_limit = { let parallelism_limit = {
let mut limit = ParallelismLimit::default(); let mut limit = ParallelismLimit::default();
limit.set_apply_limit({ limit.set_apply_limit({
let limit = local_args.value_of("parallel").unwrap().parse::<usize>().unwrap(); let limit = local_args
.value_of("parallel")
.unwrap()
.parse::<usize>()
.unwrap();
if limit == 0 { if limit == 0 {
n_targets n_targets
} else { } else {
@ -232,12 +232,10 @@ pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(
deployment.set_parallelism_limit(parallelism_limit); deployment.set_parallelism_limit(parallelism_limit);
deployment.set_evaluation_node_limit(evaluation_node_limit); deployment.set_evaluation_node_limit(evaluation_node_limit);
let (deployment, output) = tokio::join!( let (deployment, output) = tokio::join!(deployment.execute(), output.run_until_completion(),);
deployment.execute(),
output.run_until_completion(),
);
deployment?; output?; deployment?;
output?;
Ok(()) Ok(())
} }

View file

@ -1,17 +1,12 @@
use regex::Regex; use regex::Regex;
use std::collections::HashMap; use std::collections::HashMap;
use clap::{Arg, Command as ClapCommand, ArgMatches}; use clap::{Arg, ArgMatches, Command as ClapCommand};
use tokio::fs; use tokio::fs;
use crate::error::ColmenaError; use crate::error::ColmenaError;
use crate::nix::deployment::{ use crate::nix::deployment::{Deployment, Goal, Options, TargetNode};
Deployment, use crate::nix::{host::Local as LocalHost, NodeName};
Goal,
TargetNode,
Options,
};
use crate::nix::{NodeName, host::Local as LocalHost};
use crate::progress::SimpleProgressOutput; use crate::progress::SimpleProgressOutput;
use crate::util; use crate::util;
@ -89,8 +84,10 @@ pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(
let s = if local_args.is_present("node") { let s = if local_args.is_present("node") {
local_args.value_of("node").unwrap().to_owned() local_args.value_of("node").unwrap().to_owned()
} else { } else {
hostname::get().expect("Could not get hostname") hostname::get()
.to_string_lossy().into_owned() .expect("Could not get hostname")
.to_string_lossy()
.into_owned()
}; };
NodeName::new(s)? NodeName::new(s)?
@ -101,7 +98,10 @@ pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(
if let Some(info) = hive.deployment_info_single(&hostname).await.unwrap() { if let Some(info) = hive.deployment_info_single(&hostname).await.unwrap() {
let nix_options = hive.nix_options_with_builders().await.unwrap(); let nix_options = hive.nix_options_with_builders().await.unwrap();
if !info.allows_local_deployment() { if !info.allows_local_deployment() {
log::error!("Local deployment is not enabled for host {}.", hostname.as_str()); log::error!(
"Local deployment is not enabled for host {}.",
hostname.as_str()
);
log::error!("Hint: Set deployment.allowLocalDeployment to true."); log::error!("Hint: Set deployment.allowLocalDeployment to true.");
quit::with_code(2); quit::with_code(2);
} }
@ -111,13 +111,12 @@ pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(
host.set_privilege_escalation_command(Some(command)); host.set_privilege_escalation_command(Some(command));
} }
TargetNode::new( TargetNode::new(hostname.clone(), Some(host.upcast()), info.clone())
hostname.clone(),
Some(host.upcast()),
info.clone(),
)
} else { } else {
log::error!("Host \"{}\" is not present in the Hive configuration.", hostname.as_str()); log::error!(
"Host \"{}\" is not present in the Hive configuration.",
hostname.as_str()
);
quit::with_code(2); quit::with_code(2);
} }
}; };
@ -138,12 +137,10 @@ pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(
deployment.set_options(options); deployment.set_options(options);
let (deployment, output) = tokio::join!( let (deployment, output) = tokio::join!(deployment.execute(), output.run_until_completion(),);
deployment.execute(),
output.run_until_completion(),
);
deployment?; output?; deployment?;
output?;
Ok(()) Ok(())
} }

View file

@ -8,14 +8,18 @@ pub use super::apply::run;
pub fn subcommand() -> ClapCommand<'static> { pub fn subcommand() -> ClapCommand<'static> {
let command = ClapCommand::new("build") let command = ClapCommand::new("build")
.about("Build configurations but not push to remote machines") .about("Build configurations but not push to remote machines")
.long_about(r#"Build configurations but not push to remote machines .long_about(
r#"Build configurations but not push to remote machines
This subcommand behaves as if you invoked `apply` with the `build` goal."#) This subcommand behaves as if you invoked `apply` with the `build` goal."#,
.arg(Arg::new("goal") )
.hide(true) .arg(
.default_value("build") Arg::new("goal")
.possible_values(&["build"]) .hide(true)
.takes_value(true)); .default_value("build")
.possible_values(&["build"])
.takes_value(true),
);
let command = apply::register_deploy_args(command); let command = apply::register_deploy_args(command);

View file

@ -1,6 +1,6 @@
use std::path::PathBuf; use std::path::PathBuf;
use clap::{Arg, Command as ClapCommand, ArgMatches}; use clap::{Arg, ArgMatches, Command as ClapCommand};
use crate::error::ColmenaError; use crate::error::ColmenaError;
use crate::util; use crate::util;
@ -10,8 +10,7 @@ pub fn subcommand() -> ClapCommand<'static> {
} }
pub fn deprecated_alias() -> ClapCommand<'static> { pub fn deprecated_alias() -> ClapCommand<'static> {
subcommand_gen("introspect") subcommand_gen("introspect").hide(true)
.hide(true)
} }
fn subcommand_gen(name: &str) -> ClapCommand<'static> { fn subcommand_gen(name: &str) -> ClapCommand<'static> {
@ -43,7 +42,9 @@ For example, to retrieve the configuration of one node, you may write something
pub async fn run(global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(), ColmenaError> { pub async fn run(global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(), ColmenaError> {
if let Some("introspect") = global_args.subcommand_name() { if let Some("introspect") = global_args.subcommand_name() {
log::warn!("`colmena introspect` has been renamed to `colmena eval`. Please update your scripts."); log::warn!(
"`colmena introspect` has been renamed to `colmena eval`. Please update your scripts."
);
} }
let hive = util::hive_from_args(local_args).await?; let hive = util::hive_from_args(local_args).await?;
@ -57,7 +58,13 @@ pub async fn run(global_args: &ArgMatches, local_args: &ArgMatches) -> Result<()
local_args.value_of("expression").unwrap().to_string() local_args.value_of("expression").unwrap().to_string()
} else { } else {
let path: PathBuf = local_args.value_of("expression_file").unwrap().into(); let path: PathBuf = local_args.value_of("expression_file").unwrap().into();
format!("import {}", path.canonicalize().expect("Could not generate absolute path to expression file.").to_str().unwrap()) format!(
"import {}",
path.canonicalize()
.expect("Could not generate absolute path to expression file.")
.to_str()
.unwrap()
)
}; };
let instantiate = local_args.is_present("instantiate"); let instantiate = local_args.is_present("instantiate");

View file

@ -2,7 +2,7 @@ use std::env;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
use clap::{Arg, Command as ClapCommand, ArgMatches}; use clap::{Arg, ArgMatches, Command as ClapCommand};
use futures::future::join_all; use futures::future::join_all;
use tokio::sync::Semaphore; use tokio::sync::Semaphore;
@ -16,59 +16,67 @@ pub fn subcommand() -> ClapCommand<'static> {
let command = ClapCommand::new("exec") let command = ClapCommand::new("exec")
.about("Run a command on remote machines") .about("Run a command on remote machines")
.trailing_var_arg(true) .trailing_var_arg(true)
.arg(Arg::new("parallel") .arg(
.short('p') Arg::new("parallel")
.long("parallel") .short('p')
.value_name("LIMIT") .long("parallel")
.help("Deploy parallelism limit") .value_name("LIMIT")
.long_help(r#"Limits the maximum number of hosts to run the command in parallel. .help("Deploy parallelism limit")
.long_help(
r#"Limits the maximum number of hosts to run the command in parallel.
In `colmena exec`, the parallelism limit is disabled (0) by default. In `colmena exec`, the parallelism limit is disabled (0) by default.
"#) "#,
.default_value("0") )
.takes_value(true) .default_value("0")
.validator(|s| { .takes_value(true)
match s.parse::<usize>() { .validator(|s| match s.parse::<usize>() {
Ok(_) => Ok(()), Ok(_) => Ok(()),
Err(_) => Err(String::from("The value must be a valid number")), Err(_) => Err(String::from("The value must be a valid number")),
} }),
})) )
.arg(Arg::new("verbose") .arg(
.short('v') Arg::new("verbose")
.long("verbose") .short('v')
.help("Be verbose") .long("verbose")
.long_help("Deactivates the progress spinner and prints every line of output.") .help("Be verbose")
.takes_value(false)) .long_help("Deactivates the progress spinner and prints every line of output.")
.arg(Arg::new("command") .takes_value(false),
.value_name("COMMAND") )
.last(true) .arg(
.help("Command") Arg::new("command")
.required(true) .value_name("COMMAND")
.multiple_occurrences(true) .last(true)
.long_help(r#"Command to run .help("Command")
.required(true)
.multiple_occurrences(true)
.long_help(
r#"Command to run
It's recommended to use -- to separate Colmena options from the command to run. For example: It's recommended to use -- to separate Colmena options from the command to run. For example:
colmena exec --on @routers -- tcpdump -vni any ip[9] == 89 colmena exec --on @routers -- tcpdump -vni any ip[9] == 89
"#)); "#,
),
);
util::register_selector_args(command) util::register_selector_args(command)
} }
pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(), ColmenaError> { pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(), ColmenaError> {
let hive = util::hive_from_args(local_args).await?; let hive = util::hive_from_args(local_args).await?;
let ssh_config = env::var("SSH_CONFIG_FILE") let ssh_config = env::var("SSH_CONFIG_FILE").ok().map(PathBuf::from);
.ok().map(PathBuf::from);
let filter = local_args.value_of("on") let filter = local_args.value_of("on").map(NodeFilter::new).transpose()?;
.map(NodeFilter::new)
.transpose()?;
let mut targets = hive.select_nodes(filter, ssh_config, true).await?; let mut targets = hive.select_nodes(filter, ssh_config, true).await?;
let parallel_sp = Arc::new({ let parallel_sp = Arc::new({
let limit = local_args.value_of("parallel").unwrap() let limit = local_args
.parse::<usize>().unwrap(); .value_of("parallel")
.unwrap()
.parse::<usize>()
.unwrap();
if limit > 0 { if limit > 0 {
Some(Semaphore::new(limit)) Some(Semaphore::new(limit))
@ -77,7 +85,13 @@ pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(
} }
}); });
let command: Arc<Vec<String>> = Arc::new(local_args.values_of("command").unwrap().map(|s| s.to_string()).collect()); let command: Arc<Vec<String>> = Arc::new(
local_args
.values_of("command")
.unwrap()
.map(|s| s.to_string())
.collect(),
);
let mut output = SimpleProgressOutput::new(local_args.is_present("verbose")); let mut output = SimpleProgressOutput::new(local_args.is_present("verbose"));
@ -91,7 +105,7 @@ pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(
let mut host = target.into_host().unwrap(); let mut host = target.into_host().unwrap();
let job = meta.create_job(JobType::Execute, vec![ name.clone() ])?; let job = meta.create_job(JobType::Execute, vec![name.clone()])?;
futures.push(job.run_waiting(|job| async move { futures.push(job.run_waiting(|job| async move {
let permit = match parallel_sp.as_ref() { let permit = match parallel_sp.as_ref() {
@ -122,7 +136,9 @@ pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(
output.run_until_completion(), output.run_until_completion(),
); );
meta?; monitor?; output?; meta?;
monitor?;
output?;
Ok(()) Ok(())
} }

View file

@ -1,9 +1,9 @@
pub mod build;
pub mod apply; pub mod apply;
pub mod build;
pub mod eval; pub mod eval;
pub mod upload_keys;
pub mod exec; pub mod exec;
pub mod nix_info; pub mod nix_info;
pub mod upload_keys;
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
pub mod apply_local; pub mod apply_local;

View file

@ -1,12 +1,11 @@
use clap::{Command as ClapCommand, ArgMatches}; use clap::{ArgMatches, Command as ClapCommand};
use crate::error::ColmenaError; use crate::error::ColmenaError;
use crate::nix::NixCheck;
use crate::nix::evaluator::nix_eval_jobs::get_pinned_nix_eval_jobs; use crate::nix::evaluator::nix_eval_jobs::get_pinned_nix_eval_jobs;
use crate::nix::NixCheck;
pub fn subcommand() -> ClapCommand<'static> { pub fn subcommand() -> ClapCommand<'static> {
ClapCommand::new("nix-info") ClapCommand::new("nix-info").about("Show information about the current Nix installation")
.about("Show information about the current Nix installation")
} }
pub async fn run(_global_args: &ArgMatches, _local_args: &ArgMatches) -> Result<(), ColmenaError> { pub async fn run(_global_args: &ArgMatches, _local_args: &ArgMatches) -> Result<(), ColmenaError> {

View file

@ -1,17 +1,17 @@
use std::time::Duration; use std::time::Duration;
use clap::{Command as ClapCommand, ArgMatches}; use clap::{ArgMatches, Command as ClapCommand};
use tokio::time; use tokio::time;
use crate::error::{ColmenaError, ColmenaResult}; use crate::error::{ColmenaError, ColmenaResult};
use crate::job::{JobMonitor, JobType}; use crate::job::{JobMonitor, JobType};
use crate::nix::NodeName; use crate::nix::NodeName;
use crate::progress::{ProgressOutput, spinner::SpinnerOutput}; use crate::progress::{spinner::SpinnerOutput, ProgressOutput};
macro_rules! node { macro_rules! node {
($n:expr) => { ($n:expr) => {
NodeName::new($n.to_string()).unwrap() NodeName::new($n.to_string()).unwrap()
} };
} }
pub fn subcommand() -> ClapCommand<'static> { pub fn subcommand() -> ClapCommand<'static> {
@ -44,7 +44,7 @@ pub async fn run(_global_args: &ArgMatches, _local_args: &ArgMatches) -> Result<
Ok(()) Ok(())
}); });
let build = meta.create_job(JobType::Build, vec![ node!("alpha"), node!("beta") ])?; let build = meta.create_job(JobType::Build, vec![node!("alpha"), node!("beta")])?;
let build = build.run(|_| async move { let build = build.run(|_| async move {
time::sleep(Duration::from_secs(5)).await; time::sleep(Duration::from_secs(5)).await;
@ -62,7 +62,8 @@ pub async fn run(_global_args: &ArgMatches, _local_args: &ArgMatches) -> Result<
meta_future, meta_future,
); );
monitor?; output?; monitor?;
output?;
println!("Return Value -> {:?}", ret); println!("Return Value -> {:?}", ret);

View file

@ -8,14 +8,18 @@ pub use super::apply::run;
pub fn subcommand() -> ClapCommand<'static> { pub fn subcommand() -> ClapCommand<'static> {
let command = ClapCommand::new("upload-keys") let command = ClapCommand::new("upload-keys")
.about("Upload keys to remote hosts") .about("Upload keys to remote hosts")
.long_about(r#"Upload keys to remote hosts .long_about(
r#"Upload keys to remote hosts
This subcommand behaves as if you invoked `apply` with the pseudo `keys` goal."#) This subcommand behaves as if you invoked `apply` with the pseudo `keys` goal."#,
.arg(Arg::new("goal") )
.hide(true) .arg(
.default_value("keys") Arg::new("goal")
.possible_values(&["keys"]) .hide(true)
.takes_value(true)); .default_value("keys")
.possible_values(&["keys"])
.takes_value(true),
);
let command = apply::register_deploy_args(command); let command = apply::register_deploy_args(command);

View file

@ -6,7 +6,7 @@ use std::process::ExitStatus;
use snafu::Snafu; use snafu::Snafu;
use validator::ValidationErrors; use validator::ValidationErrors;
use crate::nix::{key, StorePath, Profile}; use crate::nix::{key, Profile, StorePath};
pub type ColmenaResult<T> = Result<T, ColmenaError>; pub type ColmenaResult<T> = Result<T, ColmenaError>;
@ -87,7 +87,9 @@ impl From<ExitStatus> for ColmenaError {
fn from(status: ExitStatus) -> Self { fn from(status: ExitStatus) -> Self {
match status.code() { match status.code() {
Some(exit_code) => Self::ChildFailure { exit_code }, Some(exit_code) => Self::ChildFailure { exit_code },
None => Self::ChildKilled { signal: status.signal().unwrap() }, None => Self::ChildKilled {
signal: status.signal().unwrap(),
},
} }
} }
} }

View file

@ -13,9 +13,9 @@ use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
use tokio::time; use tokio::time;
use uuid::Uuid; use uuid::Uuid;
use crate::error::{ColmenaResult, ColmenaError}; use crate::error::{ColmenaError, ColmenaResult};
use crate::nix::NodeName; use crate::nix::NodeName;
use crate::progress::{Sender as ProgressSender, Message as ProgressMessage, Line, LineStyle}; use crate::progress::{Line, LineStyle, Message as ProgressMessage, Sender as ProgressSender};
pub type Sender = UnboundedSender<Event>; pub type Sender = UnboundedSender<Event>;
pub type Receiver = UnboundedReceiver<Event>; pub type Receiver = UnboundedReceiver<Event>;
@ -311,7 +311,12 @@ impl JobMonitor {
} }
EventPayload::SuccessWithMessage(custom_message) => { EventPayload::SuccessWithMessage(custom_message) => {
let custom_message = Some(custom_message.clone()); let custom_message = Some(custom_message.clone());
self.update_job_state(message.job_id, JobState::Succeeded, custom_message, false); self.update_job_state(
message.job_id,
JobState::Succeeded,
custom_message,
false,
);
if message.job_id != self.meta_job_id { if message.job_id != self.meta_job_id {
self.print_job_stats(); self.print_job_stats();
@ -319,7 +324,12 @@ impl JobMonitor {
} }
EventPayload::Noop(custom_message) => { EventPayload::Noop(custom_message) => {
let custom_message = Some(custom_message.clone()); let custom_message = Some(custom_message.clone());
self.update_job_state(message.job_id, JobState::Succeeded, custom_message, true); self.update_job_state(
message.job_id,
JobState::Succeeded,
custom_message,
true,
);
if message.job_id != self.meta_job_id { if message.job_id != self.meta_job_id {
self.print_job_stats(); self.print_job_stats();
@ -333,7 +343,9 @@ impl JobMonitor {
self.print_job_stats(); self.print_job_stats();
} }
} }
EventPayload::ChildStdout(m) | EventPayload::ChildStderr(m) | EventPayload::Message(m) => { EventPayload::ChildStdout(m)
| EventPayload::ChildStderr(m)
| EventPayload::Message(m) => {
if let Some(sender) = &self.progress { if let Some(sender) = &self.progress {
let metadata = &self.jobs[&message.job_id]; let metadata = &self.jobs[&message.job_id];
let line = metadata.get_line(m.clone()); let line = metadata.get_line(m.clone());
@ -348,7 +360,8 @@ impl JobMonitor {
} }
/// Updates the state of a job. /// Updates the state of a job.
fn update_job_state(&mut self, fn update_job_state(
&mut self,
job_id: JobId, job_id: JobId,
new_state: JobState, new_state: JobState,
message: Option<String>, message: Option<String>,
@ -373,7 +386,9 @@ impl JobMonitor {
if new_state != JobState::Waiting { if new_state != JobState::Waiting {
if let Some(sender) = &self.progress { if let Some(sender) = &self.progress {
let text = if new_state == JobState::Succeeded { let text = if new_state == JobState::Succeeded {
metadata.custom_message.clone() metadata
.custom_message
.clone()
.or_else(|| metadata.describe_state_transition()) .or_else(|| metadata.describe_state_transition())
} else { } else {
metadata.describe_state_transition() metadata.describe_state_transition()
@ -401,8 +416,7 @@ impl JobMonitor {
if let Some(sender) = &self.progress { if let Some(sender) = &self.progress {
let stats = self.get_job_stats(); let stats = self.get_job_stats();
let text = format!("{}", stats); let text = format!("{}", stats);
let line = self.jobs[&self.meta_job_id].get_line(text) let line = self.jobs[&self.meta_job_id].get_line(text).noisy();
.noisy();
let message = ProgressMessage::PrintMeta(line); let message = ProgressMessage::PrintMeta(line);
sender.send(message).unwrap(); sender.send(message).unwrap();
} }
@ -463,10 +477,23 @@ impl JobMonitor {
for job in self.jobs.values() { for job in self.jobs.values() {
if job.state == JobState::Failed { if job.state == JobState::Failed {
let logs: Vec<&Event> = self.events.iter().filter(|e| e.job_id == job.job_id).collect(); let logs: Vec<&Event> = self
let last_logs: Vec<&Event> = logs.into_iter().rev().take(LOG_CONTEXT_LINES).rev().collect(); .events
.iter()
.filter(|e| e.job_id == job.job_id)
.collect();
let last_logs: Vec<&Event> = logs
.into_iter()
.rev()
.take(LOG_CONTEXT_LINES)
.rev()
.collect();
log::error!("{} - Last {} lines of logs:", job.get_failure_summary(), last_logs.len()); log::error!(
"{} - Last {} lines of logs:",
job.get_failure_summary(),
last_logs.len()
);
for event in last_logs { for event in last_logs {
log::error!("{}", event.payload); log::error!("{}", event.payload);
} }
@ -498,13 +525,12 @@ impl JobHandleInner {
/// This sends out a Creation message with the metadata. /// This sends out a Creation message with the metadata.
pub fn create_job(&self, job_type: JobType, nodes: Vec<NodeName>) -> ColmenaResult<JobHandle> { pub fn create_job(&self, job_type: JobType, nodes: Vec<NodeName>) -> ColmenaResult<JobHandle> {
let job_id = JobId::new(); let job_id = JobId::new();
let creation = JobCreation { let creation = JobCreation { job_type, nodes };
job_type,
nodes,
};
if job_type == JobType::Meta { if job_type == JobType::Meta {
return Err(ColmenaError::Unknown { message: "Cannot create a meta job!".to_string() }); return Err(ColmenaError::Unknown {
message: "Cannot create a meta job!".to_string(),
});
} }
let new_handle = Arc::new(Self { let new_handle = Arc::new(Self {
@ -521,8 +547,9 @@ impl JobHandleInner {
/// ///
/// This immediately transitions the state to Running. /// This immediately transitions the state to Running.
pub async fn run<F, U, T>(self: Arc<Self>, f: U) -> ColmenaResult<T> pub async fn run<F, U, T>(self: Arc<Self>, f: U) -> ColmenaResult<T>
where U: FnOnce(Arc<Self>) -> F, where
F: Future<Output = ColmenaResult<T>>, U: FnOnce(Arc<Self>) -> F,
F: Future<Output = ColmenaResult<T>>,
{ {
self.run_internal(f, true).await self.run_internal(f, true).await
} }
@ -531,8 +558,9 @@ impl JobHandleInner {
/// ///
/// This does not immediately transition the state to Running. /// This does not immediately transition the state to Running.
pub async fn run_waiting<F, U, T>(self: Arc<Self>, f: U) -> ColmenaResult<T> pub async fn run_waiting<F, U, T>(self: Arc<Self>, f: U) -> ColmenaResult<T>
where U: FnOnce(Arc<Self>) -> F, where
F: Future<Output = ColmenaResult<T>>, U: FnOnce(Arc<Self>) -> F,
F: Future<Output = ColmenaResult<T>>,
{ {
self.run_internal(f, false).await self.run_internal(f, false).await
} }
@ -574,8 +602,9 @@ impl JobHandleInner {
/// Runs a closure, automatically updating the job monitor based on the result. /// Runs a closure, automatically updating the job monitor based on the result.
async fn run_internal<F, U, T>(self: Arc<Self>, f: U, report_running: bool) -> ColmenaResult<T> async fn run_internal<F, U, T>(self: Arc<Self>, f: U, report_running: bool) -> ColmenaResult<T>
where U: FnOnce(Arc<Self>) -> F, where
F: Future<Output = ColmenaResult<T>>, U: FnOnce(Arc<Self>) -> F,
F: Future<Output = ColmenaResult<T>>,
{ {
if report_running { if report_running {
// Tell monitor we are starting // Tell monitor we are starting
@ -606,7 +635,8 @@ impl JobHandleInner {
let event = Event::new(self.job_id, payload); let event = Event::new(self.job_id, payload);
if let Some(sender) = &self.sender { if let Some(sender) = &self.sender {
sender.send(event) sender
.send(event)
.map_err(|e| ColmenaError::unknown(Box::new(e)))?; .map_err(|e| ColmenaError::unknown(Box::new(e)))?;
} else { } else {
log::debug!("Sending event: {:?}", event); log::debug!("Sending event: {:?}", event);
@ -619,8 +649,9 @@ impl JobHandleInner {
impl MetaJobHandle { impl MetaJobHandle {
/// Runs a closure, automatically updating the job monitor based on the result. /// Runs a closure, automatically updating the job monitor based on the result.
pub async fn run<F, U, T>(self, f: U) -> ColmenaResult<T> pub async fn run<F, U, T>(self, f: U) -> ColmenaResult<T>
where U: FnOnce(JobHandle) -> F, where
F: Future<Output = ColmenaResult<T>>, U: FnOnce(JobHandle) -> F,
F: Future<Output = ColmenaResult<T>>,
{ {
let normal_handle = Arc::new(JobHandleInner { let normal_handle = Arc::new(JobHandleInner {
job_id: self.job_id, job_id: self.job_id,
@ -647,7 +678,8 @@ impl MetaJobHandle {
fn send_payload(&self, payload: EventPayload) -> ColmenaResult<()> { fn send_payload(&self, payload: EventPayload) -> ColmenaResult<()> {
let event = Event::new(self.job_id, payload); let event = Event::new(self.job_id, payload);
self.sender.send(event) self.sender
.send(event)
.map_err(|e| ColmenaError::unknown(Box::new(e)))?; .map_err(|e| ColmenaError::unknown(Box::new(e)))?;
Ok(()) Ok(())
@ -685,11 +717,10 @@ impl JobMetadata {
return None; return None;
} }
let node_list = describe_node_list(&self.nodes) let node_list =
.unwrap_or_else(|| "some node(s)".to_string()); describe_node_list(&self.nodes).unwrap_or_else(|| "some node(s)".to_string());
let message = self.custom_message.as_deref() let message = self.custom_message.as_deref().unwrap_or("No message");
.unwrap_or("No message");
Some(match (self.job_type, self.state) { Some(match (self.job_type, self.state) {
(JobType::Meta, JobState::Succeeded) => "All done!".to_string(), (JobType::Meta, JobState::Succeeded) => "All done!".to_string(),
@ -725,8 +756,8 @@ impl JobMetadata {
/// Returns a human-readable string describing a failed job for use in the summary. /// Returns a human-readable string describing a failed job for use in the summary.
fn get_failure_summary(&self) -> String { fn get_failure_summary(&self) -> String {
let node_list = describe_node_list(&self.nodes) let node_list =
.unwrap_or_else(|| "some node(s)".to_string()); describe_node_list(&self.nodes).unwrap_or_else(|| "some node(s)".to_string());
match self.job_type { match self.job_type {
JobType::Evaluate => format!("Failed to evaluate {}", node_list), JobType::Evaluate => format!("Failed to evaluate {}", node_list),
@ -757,15 +788,15 @@ impl EventPayload {
impl Display for EventPayload { impl Display for EventPayload {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self { match self {
EventPayload::ChildStdout(o) => write!(f, " stdout) {}", o)?, EventPayload::ChildStdout(o) => write!(f, " stdout) {}", o)?,
EventPayload::ChildStderr(o) => write!(f, " stderr) {}", o)?, EventPayload::ChildStderr(o) => write!(f, " stderr) {}", o)?,
EventPayload::Message(m) => write!(f, " message) {}", m)?, EventPayload::Message(m) => write!(f, " message) {}", m)?,
EventPayload::Creation(_) => write!(f, " created)")?, EventPayload::Creation(_) => write!(f, " created)")?,
EventPayload::NewState(s) => write!(f, " state) {:?}", s)?, EventPayload::NewState(s) => write!(f, " state) {:?}", s)?,
EventPayload::SuccessWithMessage(m) => write!(f, " success) {}", m)?, EventPayload::SuccessWithMessage(m) => write!(f, " success) {}", m)?,
EventPayload::Noop(m) => write!(f, " noop) {}", m)?, EventPayload::Noop(m) => write!(f, " noop) {}", m)?,
EventPayload::Failure(e) => write!(f, " failure) {}", e)?, EventPayload::Failure(e) => write!(f, " failure) {}", e)?,
EventPayload::ShutdownMonitor => write!(f, "shutdown)")?, EventPayload::ShutdownMonitor => write!(f, "shutdown)")?,
} }
Ok(()) Ok(())
@ -865,7 +896,7 @@ mod tests {
macro_rules! node { macro_rules! node {
($n:expr) => { ($n:expr) => {
NodeName::new($n.to_string()).unwrap() NodeName::new($n.to_string()).unwrap()
} };
} }
#[test] #[test]
@ -876,21 +907,20 @@ mod tests {
let meta = meta.run(|job: JobHandle| async move { let meta = meta.run(|job: JobHandle| async move {
job.message("hello world".to_string())?; job.message("hello world".to_string())?;
let eval_job = job.create_job(JobType::Evaluate, vec![ node!("alpha") ])?; let eval_job = job.create_job(JobType::Evaluate, vec![node!("alpha")])?;
eval_job.run(|job| async move { eval_job
job.stdout("child stdout".to_string())?; .run(|job| async move {
job.stdout("child stdout".to_string())?;
Ok(()) Ok(())
}).await?; })
.await?;
Err(ColmenaError::Unsupported) as ColmenaResult<()> Err(ColmenaError::Unsupported) as ColmenaResult<()>
}); });
// Run until completion // Run until completion
let (ret, monitor) = tokio::join!( let (ret, monitor) = tokio::join!(meta, monitor.run_until_completion(),);
meta,
monitor.run_until_completion(),
);
match ret { match ret {
Err(ColmenaError::Unsupported) => (), Err(ColmenaError::Unsupported) => (),

View file

@ -1,11 +1,11 @@
#![deny(unused_must_use)] #![deny(unused_must_use)]
mod error;
mod nix;
mod cli; mod cli;
mod command; mod command;
mod progress; mod error;
mod job; mod job;
mod nix;
mod progress;
mod troubleshooter; mod troubleshooter;
mod util; mod util;

View file

@ -8,7 +8,7 @@ pub mod limits;
pub use limits::{EvaluationNodeLimit, ParallelismLimit}; pub use limits::{EvaluationNodeLimit, ParallelismLimit};
pub mod options; pub mod options;
pub use options::{Options, Evaluator}; pub use options::{Evaluator, Options};
use std::collections::HashMap; use std::collections::HashMap;
use std::mem; use std::mem;
@ -18,30 +18,17 @@ use futures::future::join_all;
use itertools::Itertools; use itertools::Itertools;
use tokio_stream::StreamExt; use tokio_stream::StreamExt;
use crate::progress::Sender as ProgressSender;
use crate::job::{JobMonitor, JobHandle, JobType, JobState};
use crate::util;
use super::NixOptions; use super::NixOptions;
use crate::job::{JobHandle, JobMonitor, JobState, JobType};
use crate::progress::Sender as ProgressSender;
use crate::util;
use super::{ use super::{
Hive, evaluator::{DrvSetEvaluator, EvalError, NixEvalJobs},
Host,
NodeName,
NodeConfig,
ColmenaError,
ColmenaResult,
Profile,
ProfileDerivation,
CopyDirection,
CopyOptions,
RebootOptions,
host::Local as LocalHost, host::Local as LocalHost,
key::{Key, UploadAt as UploadKeyAt}, key::{Key, UploadAt as UploadKeyAt},
evaluator::{ ColmenaError, ColmenaResult, CopyDirection, CopyOptions, Hive, Host, NodeConfig, NodeName,
DrvSetEvaluator, Profile, ProfileDerivation, RebootOptions,
NixEvalJobs,
EvalError,
},
}; };
/// A deployment. /// A deployment.
@ -106,7 +93,12 @@ impl TargetNode {
impl Deployment { impl Deployment {
/// Creates a new deployment. /// Creates a new deployment.
pub fn new(hive: Hive, targets: TargetNodeMap, goal: Goal, progress: Option<ProgressSender>) -> Self { pub fn new(
hive: Hive,
targets: TargetNodeMap,
goal: Goal,
progress: Option<ProgressSender>,
) -> Self {
Self { Self {
hive, hive,
goal, goal,
@ -151,16 +143,15 @@ impl Deployment {
futures.push(deployment.upload_keys_to_node(meta.clone(), target)); futures.push(deployment.upload_keys_to_node(meta.clone(), target));
} }
join_all(futures).await join_all(futures)
.into_iter().collect::<ColmenaResult<Vec<()>>>()?; .await
.into_iter()
.collect::<ColmenaResult<Vec<()>>>()?;
Ok(()) Ok(())
}); });
let (result, _) = tokio::join!( let (result, _) = tokio::join!(meta_future, monitor.run_until_completion(),);
meta_future,
monitor.run_until_completion(),
);
result?; result?;
@ -183,10 +174,7 @@ impl Deployment {
Ok(()) Ok(())
}); });
let (result, _) = tokio::join!( let (result, _) = tokio::join!(meta_future, monitor.run_until_completion(),);
meta_future,
monitor.run_until_completion(),
);
result?; result?;
@ -207,10 +195,14 @@ impl Deployment {
} }
/// Executes the deployment on selected nodes, evaluating a chunk at a time. /// Executes the deployment on selected nodes, evaluating a chunk at a time.
async fn execute_chunked(self: &DeploymentHandle, parent: JobHandle, mut targets: TargetNodeMap) async fn execute_chunked(
-> ColmenaResult<()> self: &DeploymentHandle,
{ parent: JobHandle,
let eval_limit = self.evaluation_node_limit.get_limit() mut targets: TargetNodeMap,
) -> ColmenaResult<()> {
let eval_limit = self
.evaluation_node_limit
.get_limit()
.unwrap_or(self.targets.len()); .unwrap_or(self.targets.len());
let mut futures = Vec::new(); let mut futures = Vec::new();
@ -224,7 +216,8 @@ impl Deployment {
futures.push(self.execute_one_chunk(parent.clone(), map)); futures.push(self.execute_one_chunk(parent.clone(), map));
} }
join_all(futures).await join_all(futures)
.await
.into_iter() .into_iter()
.collect::<ColmenaResult<Vec<()>>>()?; .collect::<ColmenaResult<Vec<()>>>()?;
@ -232,9 +225,11 @@ impl Deployment {
} }
/// Executes the deployment on selected nodes using a streaming evaluator. /// Executes the deployment on selected nodes using a streaming evaluator.
async fn execute_streaming(self: &DeploymentHandle, parent: JobHandle, mut targets: TargetNodeMap) async fn execute_streaming(
-> ColmenaResult<()> self: &DeploymentHandle,
{ parent: JobHandle,
mut targets: TargetNodeMap,
) -> ColmenaResult<()> {
if self.goal == Goal::UploadKeys { if self.goal == Goal::UploadKeys {
unreachable!(); // some logic is screwed up unreachable!(); // some logic is screwed up
} }
@ -244,81 +239,101 @@ impl Deployment {
let job = parent.create_job(JobType::Evaluate, nodes.clone())?; let job = parent.create_job(JobType::Evaluate, nodes.clone())?;
let futures = job.run(|job| async move { let futures = job
let mut evaluator = NixEvalJobs::default(); .run(|job| async move {
let eval_limit = self.evaluation_node_limit.get_limit().unwrap_or(self.targets.len()); let mut evaluator = NixEvalJobs::default();
evaluator.set_eval_limit(eval_limit); let eval_limit = self
evaluator.set_job(job.clone()); .evaluation_node_limit
.get_limit()
.unwrap_or(self.targets.len());
evaluator.set_eval_limit(eval_limit);
evaluator.set_job(job.clone());
// FIXME: nix-eval-jobs currently does not support IFD with builders // FIXME: nix-eval-jobs currently does not support IFD with builders
let options = self.hive.nix_options(); let options = self.hive.nix_options();
let mut stream = evaluator.evaluate(&expr, options).await?; let mut stream = evaluator.evaluate(&expr, options).await?;
let mut futures: Vec<tokio::task::JoinHandle<ColmenaResult<()>>> = Vec::new(); let mut futures: Vec<tokio::task::JoinHandle<ColmenaResult<()>>> = Vec::new();
while let Some(item) = stream.next().await { while let Some(item) = stream.next().await {
match item { match item {
Ok(attr) => { Ok(attr) => {
let node_name = NodeName::new(attr.attribute().to_owned())?; let node_name = NodeName::new(attr.attribute().to_owned())?;
let profile_drv: ProfileDerivation = attr.into_derivation()?; let profile_drv: ProfileDerivation = attr.into_derivation()?;
// FIXME: Consolidate // FIXME: Consolidate
let mut target = targets.remove(&node_name).unwrap(); let mut target = targets.remove(&node_name).unwrap();
if let Some(force_build_on_target) = self.options.force_build_on_target { if let Some(force_build_on_target) = self.options.force_build_on_target
target.config.set_build_on_target(force_build_on_target); {
} target.config.set_build_on_target(force_build_on_target);
}
let job_handle = job.clone(); let job_handle = job.clone();
let arc_self = self.clone(); let arc_self = self.clone();
futures.push(tokio::spawn(async move { futures.push(tokio::spawn(async move {
let (target, profile) = { let (target, profile) = {
if target.config.build_on_target() { if target.config.build_on_target() {
arc_self.build_on_node(job_handle.clone(), target, profile_drv.clone()).await? arc_self
.build_on_node(
job_handle.clone(),
target,
profile_drv.clone(),
)
.await?
} else {
arc_self
.build_and_push_node(
job_handle.clone(),
target,
profile_drv.clone(),
)
.await?
}
};
if arc_self.goal.requires_activation() {
arc_self.activate_node(job_handle, target, profile).await
} else { } else {
arc_self.build_and_push_node(job_handle.clone(), target, profile_drv.clone()).await? Ok(())
} }
}; }));
}
if arc_self.goal.requires_activation() { Err(e) => {
arc_self.activate_node(job_handle, target, profile).await match e {
} else { EvalError::Global(e) => {
Ok(()) // Global error - Abort immediately
} return Err(e);
})); }
} EvalError::Attribute(e) => {
Err(e) => { // Attribute-level error
match e { //
EvalError::Global(e) => { // Here the eventual non-zero exit code of the evaluator
// Global error - Abort immediately // will translate into an `EvalError::Global`, causing
return Err(e); // the entire future to resolve to an Err.
}
EvalError::Attribute(e) => { let node_name =
// Attribute-level error NodeName::new(e.attribute().to_string()).unwrap();
// let nodes = vec![node_name];
// Here the eventual non-zero exit code of the evaluator let job = parent.create_job(JobType::Evaluate, nodes)?;
// will translate into an `EvalError::Global`, causing
// the entire future to resolve to an Err. job.state(JobState::Running)?;
for line in e.error().lines() {
let node_name = NodeName::new(e.attribute().to_string()).unwrap(); job.stderr(line.to_string())?;
let nodes = vec![ node_name ]; }
let job = parent.create_job(JobType::Evaluate, nodes)?; job.state(JobState::Failed)?;
job.state(JobState::Running)?;
for line in e.error().lines() {
job.stderr(line.to_string())?;
} }
job.state(JobState::Failed)?;
} }
} }
} }
} }
}
Ok(futures) Ok(futures)
}).await?; })
.await?;
join_all(futures).await join_all(futures)
.await
.into_iter() .into_iter()
.map(|r| r.unwrap()) // panic on JoinError (future panicked) .map(|r| r.unwrap()) // panic on JoinError (future panicked)
.collect::<ColmenaResult<Vec<()>>>()?; .collect::<ColmenaResult<Vec<()>>>()?;
@ -327,7 +342,11 @@ impl Deployment {
} }
/// Executes the deployment against a portion of nodes. /// Executes the deployment against a portion of nodes.
async fn execute_one_chunk(self: &DeploymentHandle, parent: JobHandle, mut chunk: TargetNodeMap) -> ColmenaResult<()> { async fn execute_one_chunk(
self: &DeploymentHandle,
parent: JobHandle,
mut chunk: TargetNodeMap,
) -> ColmenaResult<()> {
if self.goal == Goal::UploadKeys { if self.goal == Goal::UploadKeys {
unreachable!(); // some logic is screwed up unreachable!(); // some logic is screwed up
} }
@ -349,9 +368,13 @@ impl Deployment {
futures.push(async move { futures.push(async move {
let (target, profile) = { let (target, profile) = {
if target.config.build_on_target() { if target.config.build_on_target() {
arc_self.build_on_node(job_handle.clone(), target, profile_drv.clone()).await? arc_self
.build_on_node(job_handle.clone(), target, profile_drv.clone())
.await?
} else { } else {
arc_self.build_and_push_node(job_handle.clone(), target, profile_drv.clone()).await? arc_self
.build_and_push_node(job_handle.clone(), target, profile_drv.clone())
.await?
} }
}; };
@ -363,16 +386,20 @@ impl Deployment {
}); });
} }
join_all(futures).await join_all(futures)
.into_iter().collect::<ColmenaResult<Vec<()>>>()?; .await
.into_iter()
.collect::<ColmenaResult<Vec<()>>>()?;
Ok(()) Ok(())
} }
/// Evaluates a set of nodes, returning their corresponding store derivations. /// Evaluates a set of nodes, returning their corresponding store derivations.
async fn evaluate_nodes(self: &DeploymentHandle, parent: JobHandle, nodes: Vec<NodeName>) async fn evaluate_nodes(
-> ColmenaResult<HashMap<NodeName, ProfileDerivation>> self: &DeploymentHandle,
{ parent: JobHandle,
nodes: Vec<NodeName>,
) -> ColmenaResult<HashMap<NodeName, ProfileDerivation>> {
let job = parent.create_job(JobType::Evaluate, nodes.clone())?; let job = parent.create_job(JobType::Evaluate, nodes.clone())?;
job.run_waiting(|job| async move { job.run_waiting(|job| async move {
@ -384,11 +411,16 @@ impl Deployment {
drop(permit); drop(permit);
result result
}).await })
.await
} }
/// Only uploads keys to a node. /// Only uploads keys to a node.
async fn upload_keys_to_node(self: &DeploymentHandle, parent: JobHandle, mut target: TargetNode) -> ColmenaResult<()> { async fn upload_keys_to_node(
self: &DeploymentHandle,
parent: JobHandle,
mut target: TargetNode,
) -> ColmenaResult<()> {
let nodes = vec![target.name.clone()]; let nodes = vec![target.name.clone()];
let job = parent.create_job(JobType::UploadKeys, nodes)?; let job = parent.create_job(JobType::UploadKeys, nodes)?;
job.run(|_| async move { job.run(|_| async move {
@ -400,37 +432,44 @@ impl Deployment {
host.upload_keys(&target.config.keys, true).await?; host.upload_keys(&target.config.keys, true).await?;
Ok(()) Ok(())
}).await })
.await
} }
/// Builds a system profile directly on the node itself. /// Builds a system profile directly on the node itself.
async fn build_on_node(self: &DeploymentHandle, parent: JobHandle, mut target: TargetNode, profile_drv: ProfileDerivation) async fn build_on_node(
-> ColmenaResult<(TargetNode, Profile)> self: &DeploymentHandle,
{ parent: JobHandle,
mut target: TargetNode,
profile_drv: ProfileDerivation,
) -> ColmenaResult<(TargetNode, Profile)> {
let nodes = vec![target.name.clone()]; let nodes = vec![target.name.clone()];
let permit = self.parallelism_limit.apply.acquire().await.unwrap(); let permit = self.parallelism_limit.apply.acquire().await.unwrap();
let build_job = parent.create_job(JobType::Build, nodes.clone())?; let build_job = parent.create_job(JobType::Build, nodes.clone())?;
let (target, profile) = build_job.run(|job| async move { let (target, profile) = build_job
if target.host.is_none() { .run(|job| async move {
return Err(ColmenaError::Unsupported); if target.host.is_none() {
} return Err(ColmenaError::Unsupported);
}
let host = target.host.as_mut().unwrap(); let host = target.host.as_mut().unwrap();
host.set_job(Some(job.clone())); host.set_job(Some(job.clone()));
host.copy_closure( host.copy_closure(
profile_drv.as_store_path(), profile_drv.as_store_path(),
CopyDirection::ToRemote, CopyDirection::ToRemote,
CopyOptions::default().include_outputs(true), CopyOptions::default().include_outputs(true),
).await?; )
.await?;
let profile = profile_drv.realize_remote(host).await?; let profile = profile_drv.realize_remote(host).await?;
job.success_with_message(format!("Built {:?} on target node", profile.as_path()))?; job.success_with_message(format!("Built {:?} on target node", profile.as_path()))?;
Ok((target, profile)) Ok((target, profile))
}).await?; })
.await?;
drop(permit); drop(permit);
@ -438,9 +477,12 @@ impl Deployment {
} }
/// Builds and pushes a system profile on a node. /// Builds and pushes a system profile on a node.
async fn build_and_push_node(self: &DeploymentHandle, parent: JobHandle, target: TargetNode, profile_drv: ProfileDerivation) async fn build_and_push_node(
-> ColmenaResult<(TargetNode, Profile)> self: &DeploymentHandle,
{ parent: JobHandle,
target: TargetNode,
profile_drv: ProfileDerivation,
) -> ColmenaResult<(TargetNode, Profile)> {
let nodes = vec![target.name.clone()]; let nodes = vec![target.name.clone()];
let permit = self.parallelism_limit.apply.acquire().await.unwrap(); let permit = self.parallelism_limit.apply.acquire().await.unwrap();
@ -448,16 +490,18 @@ impl Deployment {
// Build system profile // Build system profile
let build_job = parent.create_job(JobType::Build, nodes.clone())?; let build_job = parent.create_job(JobType::Build, nodes.clone())?;
let arc_self = self.clone(); let arc_self = self.clone();
let profile: Profile = build_job.run(|job| async move { let profile: Profile = build_job
// FIXME: Remote builder? .run(|job| async move {
let mut builder = LocalHost::new(arc_self.nix_options.clone()).upcast(); // FIXME: Remote builder?
builder.set_job(Some(job.clone())); let mut builder = LocalHost::new(arc_self.nix_options.clone()).upcast();
builder.set_job(Some(job.clone()));
let profile = profile_drv.realize(&mut builder).await?; let profile = profile_drv.realize(&mut builder).await?;
job.success_with_message(format!("Built {:?}", profile.as_path()))?; job.success_with_message(format!("Built {:?}", profile.as_path()))?;
Ok(profile) Ok(profile)
}).await?; })
.await?;
// Create GC root // Create GC root
let profile_r = profile.clone(); let profile_r = profile.clone();
@ -474,7 +518,8 @@ impl Deployment {
job.noop("No context directory to create GC roots in".to_string())?; job.noop("No context directory to create GC roots in".to_string())?;
} }
Ok(target) Ok(target)
}).await? })
.await?
} else { } else {
target target
}; };
@ -487,20 +532,24 @@ impl Deployment {
let push_job = parent.create_job(JobType::Push, nodes.clone())?; let push_job = parent.create_job(JobType::Push, nodes.clone())?;
let push_profile = profile.clone(); let push_profile = profile.clone();
let arc_self = self.clone(); let arc_self = self.clone();
let target = push_job.run(|job| async move { let target = push_job
if target.host.is_none() { .run(|job| async move {
return Err(ColmenaError::Unsupported); if target.host.is_none() {
} return Err(ColmenaError::Unsupported);
}
let host = target.host.as_mut().unwrap(); let host = target.host.as_mut().unwrap();
host.set_job(Some(job.clone())); host.set_job(Some(job.clone()));
host.copy_closure( host.copy_closure(
push_profile.as_store_path(), push_profile.as_store_path(),
CopyDirection::ToRemote, CopyDirection::ToRemote,
arc_self.options.to_copy_options()).await?; arc_self.options.to_copy_options(),
)
.await?;
Ok(target) Ok(target)
}).await?; })
.await?;
drop(permit); drop(permit);
@ -510,9 +559,12 @@ impl Deployment {
/// Activates a system profile on a node. /// Activates a system profile on a node.
/// ///
/// This will also upload keys to the node. /// This will also upload keys to the node.
async fn activate_node(self: DeploymentHandle, parent: JobHandle, mut target: TargetNode, profile: Profile) async fn activate_node(
-> ColmenaResult<()> self: DeploymentHandle,
{ parent: JobHandle,
mut target: TargetNode,
profile: Profile,
) -> ColmenaResult<()> {
let nodes = vec![target.name.clone()]; let nodes = vec![target.name.clone()];
let permit = self.parallelism_limit.apply.acquire().await.unwrap(); let permit = self.parallelism_limit.apply.acquire().await.unwrap();
@ -521,7 +573,10 @@ impl Deployment {
let mut target = if self.options.upload_keys { let mut target = if self.options.upload_keys {
let job = parent.create_job(JobType::UploadKeys, nodes.clone())?; let job = parent.create_job(JobType::UploadKeys, nodes.clone())?;
job.run_waiting(|job| async move { job.run_waiting(|job| async move {
let keys = target.config.keys.iter() let keys = target
.config
.keys
.iter()
.filter(|(_, v)| v.upload_at() == UploadKeyAt::PreActivation) .filter(|(_, v)| v.upload_at() == UploadKeyAt::PreActivation)
.map(|(k, v)| (k.clone(), v.clone())) .map(|(k, v)| (k.clone(), v.clone()))
.collect::<HashMap<String, Key>>(); .collect::<HashMap<String, Key>>();
@ -540,7 +595,8 @@ impl Deployment {
job.success_with_message("Uploaded keys (pre-activation)".to_string())?; job.success_with_message("Uploaded keys (pre-activation)".to_string())?;
Ok(target) Ok(target)
}).await? })
.await?
} else { } else {
target target
}; };
@ -580,7 +636,10 @@ impl Deployment {
let mut target = if self.options.upload_keys { let mut target = if self.options.upload_keys {
let job = parent.create_job(JobType::UploadKeys, nodes.clone())?; let job = parent.create_job(JobType::UploadKeys, nodes.clone())?;
job.run_waiting(|job| async move { job.run_waiting(|job| async move {
let keys = target.config.keys.iter() let keys = target
.config
.keys
.iter()
.filter(|(_, v)| v.upload_at() == UploadKeyAt::PostActivation) .filter(|(_, v)| v.upload_at() == UploadKeyAt::PostActivation)
.map(|(k, v)| (k.clone(), v.clone())) .map(|(k, v)| (k.clone(), v.clone()))
.collect::<HashMap<String, Key>>(); .collect::<HashMap<String, Key>>();
@ -599,7 +658,8 @@ impl Deployment {
job.success_with_message("Uploaded keys (post-activation)".to_string())?; job.success_with_message("Uploaded keys (post-activation)".to_string())?;
Ok(target) Ok(target)
}).await? })
.await?
} else { } else {
target target
}; };
@ -625,7 +685,8 @@ impl Deployment {
host.reboot(options).await?; host.reboot(options).await?;
Ok(()) Ok(())
}).await?; })
.await?;
} }
drop(permit); drop(permit);

View file

@ -116,6 +116,6 @@ impl FromStr for Evaluator {
impl Evaluator { impl Evaluator {
pub fn possible_values() -> &'static [&'static str] { pub fn possible_values() -> &'static [&'static str] {
&[ "chunked", "streaming" ] &["chunked", "streaming"]
} }
} }

View file

@ -17,9 +17,9 @@ use std::result::Result as StdResult;
use async_trait::async_trait; use async_trait::async_trait;
use futures::Stream; use futures::Stream;
use super::{BuildResult, NixExpression, NixOptions, StoreDerivation, StorePath};
use crate::error::{ColmenaError, ColmenaResult};
use crate::job::JobHandle; use crate::job::JobHandle;
use crate::error::{ColmenaResult, ColmenaError};
use super::{BuildResult, StorePath, StoreDerivation, NixExpression, NixOptions};
/// The result of an evaluation. /// The result of an evaluation.
/// ///
@ -58,7 +58,11 @@ pub struct AttributeError {
#[async_trait] #[async_trait]
pub trait DrvSetEvaluator { pub trait DrvSetEvaluator {
/// Evaluates an attribute set of derivation, returning results as they come in. /// Evaluates an attribute set of derivation, returning results as they come in.
async fn evaluate(&self, expression: &dyn NixExpression, options: NixOptions) -> ColmenaResult<Pin<Box<dyn Stream<Item = EvalResult>>>>; async fn evaluate(
&self,
expression: &dyn NixExpression,
options: NixOptions,
) -> ColmenaResult<Pin<Box<dyn Stream<Item = EvalResult>>>>;
/// Sets the maximum number of attributes to evaluate at the same time. /// Sets the maximum number of attributes to evaluate at the same time.
#[allow(unused_variables)] #[allow(unused_variables)]
@ -77,7 +81,8 @@ impl AttributeOutput {
/// Returns the derivation for this attribute. /// Returns the derivation for this attribute.
pub fn into_derivation<T>(self) -> ColmenaResult<StoreDerivation<T>> pub fn into_derivation<T>(self) -> ColmenaResult<StoreDerivation<T>>
where T: TryFrom<BuildResult<T>>, where
T: TryFrom<BuildResult<T>>,
{ {
self.drv_path.into_derivation() self.drv_path.into_derivation()
} }

View file

@ -18,11 +18,11 @@ use tempfile::NamedTempFile;
use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::process::Command; use tokio::process::Command;
use crate::error::{ColmenaResult, ColmenaError}; use super::{AttributeError, AttributeOutput, DrvSetEvaluator, EvalError, EvalResult};
use crate::job::{JobHandle, null_job_handle}; use crate::error::{ColmenaError, ColmenaResult};
use crate::nix::{StorePath, NixExpression, NixOptions}; use crate::job::{null_job_handle, JobHandle};
use crate::nix::{NixExpression, NixOptions, StorePath};
use crate::util::capture_stream; use crate::util::capture_stream;
use super::{DrvSetEvaluator, EvalResult, EvalError, AttributeOutput, AttributeError};
/// The pinned nix-eval-jobs binary. /// The pinned nix-eval-jobs binary.
pub const NIX_EVAL_JOBS: Option<&str> = option_env!("NIX_EVAL_JOBS"); pub const NIX_EVAL_JOBS: Option<&str> = option_env!("NIX_EVAL_JOBS");
@ -73,7 +73,11 @@ struct EvalLineGlobalError {
#[async_trait] #[async_trait]
impl DrvSetEvaluator for NixEvalJobs { impl DrvSetEvaluator for NixEvalJobs {
async fn evaluate(&self, expression: &dyn NixExpression, options: NixOptions) -> ColmenaResult<Pin<Box<dyn Stream<Item = EvalResult>>>> { async fn evaluate(
&self,
expression: &dyn NixExpression,
options: NixOptions,
) -> ColmenaResult<Pin<Box<dyn Stream<Item = EvalResult>>>> {
let expr_file = { let expr_file = {
let mut f = NamedTempFile::new()?; let mut f = NamedTempFile::new()?;
f.write_all(expression.expression().as_bytes())?; f.write_all(expression.expression().as_bytes())?;
@ -83,7 +87,8 @@ impl DrvSetEvaluator for NixEvalJobs {
let mut command = Command::new(&self.executable); let mut command = Command::new(&self.executable);
command command
.arg("--impure") .arg("--impure")
.arg("--workers").arg(self.workers.to_string()) .arg("--workers")
.arg(self.workers.to_string())
.arg(&expr_file); .arg(&expr_file);
command.args(options.to_args()); command.args(options.to_args());
@ -101,9 +106,7 @@ impl DrvSetEvaluator for NixEvalJobs {
let stderr = BufReader::new(child.stderr.take().unwrap()); let stderr = BufReader::new(child.stderr.take().unwrap());
let job = self.job.clone(); let job = self.job.clone();
tokio::spawn(async move { tokio::spawn(async move { capture_stream(stderr, Some(job), true).await });
capture_stream(stderr, Some(job), true).await
});
Ok(Box::pin(stream! { Ok(Box::pin(stream! {
loop { loop {
@ -206,9 +209,7 @@ impl From<EvalLineAttributeError> for AttributeError {
impl From<EvalLineGlobalError> for ColmenaError { impl From<EvalLineGlobalError> for ColmenaError {
fn from(ele: EvalLineGlobalError) -> Self { fn from(ele: EvalLineGlobalError) -> Self {
ColmenaError::Unknown { ColmenaError::Unknown { message: ele.error }
message: ele.error,
}
} }
} }
@ -234,8 +235,8 @@ mod tests {
use super::*; use super::*;
use ntest::timeout; use ntest::timeout;
use tokio_test::block_on;
use tokio_stream::StreamExt; use tokio_stream::StreamExt;
use tokio_test::block_on;
#[test] #[test]
#[timeout(30000)] #[timeout(30000)]
@ -244,7 +245,10 @@ mod tests {
let expr = r#"with import <nixpkgs> {}; { a = pkgs.hello; b = pkgs.bash; }"#.to_string(); let expr = r#"with import <nixpkgs> {}; { a = pkgs.hello; b = pkgs.bash; }"#.to_string();
block_on(async move { block_on(async move {
let mut stream = evaluator.evaluate(&expr, NixOptions::default()).await.unwrap(); let mut stream = evaluator
.evaluate(&expr, NixOptions::default())
.await
.unwrap();
let mut count = 0; let mut count = 0;
while let Some(value) = stream.next().await { while let Some(value) = stream.next().await {
@ -265,7 +269,10 @@ mod tests {
let expr = r#"gibberish"#.to_string(); let expr = r#"gibberish"#.to_string();
block_on(async move { block_on(async move {
let mut stream = evaluator.evaluate(&expr, NixOptions::default()).await.unwrap(); let mut stream = evaluator
.evaluate(&expr, NixOptions::default())
.await
.unwrap();
let mut count = 0; let mut count = 0;
while let Some(value) = stream.next().await { while let Some(value) = stream.next().await {
@ -282,10 +289,14 @@ mod tests {
#[timeout(30000)] #[timeout(30000)]
fn test_attribute_error() { fn test_attribute_error() {
let evaluator = NixEvalJobs::default(); let evaluator = NixEvalJobs::default();
let expr = r#"with import <nixpkgs> {}; { a = pkgs.hello; b = throw "an error"; }"#.to_string(); let expr =
r#"with import <nixpkgs> {}; { a = pkgs.hello; b = throw "an error"; }"#.to_string();
block_on(async move { block_on(async move {
let mut stream = evaluator.evaluate(&expr, NixOptions::default()).await.unwrap(); let mut stream = evaluator
.evaluate(&expr, NixOptions::default())
.await
.unwrap();
let mut count = 0; let mut count = 0;
while let Some(value) = stream.next().await { while let Some(value) = stream.next().await {
@ -295,16 +306,14 @@ mod tests {
Ok(v) => { Ok(v) => {
assert_eq!("a", v.attribute); assert_eq!("a", v.attribute);
} }
Err(e) => { Err(e) => match e {
match e { EvalError::Attribute(a) => {
EvalError::Attribute(a) => { assert_eq!("b", a.attribute);
assert_eq!("b", a.attribute);
}
_ => {
panic!("Expected an attribute error, got {:?}", e);
}
} }
} _ => {
panic!("Expected an attribute error, got {:?}", e);
}
},
} }
count += 1; count += 1;
} }
@ -324,7 +333,10 @@ mod tests {
let expr = r#"with import <nixpkgs> {}; { a = pkgs.hello; b = pkgs.writeText "x" (import /sys/nonexistentfile); }"#.to_string(); let expr = r#"with import <nixpkgs> {}; { a = pkgs.hello; b = pkgs.writeText "x" (import /sys/nonexistentfile); }"#.to_string();
block_on(async move { block_on(async move {
let mut stream = evaluator.evaluate(&expr, NixOptions::default()).await.unwrap(); let mut stream = evaluator
.evaluate(&expr, NixOptions::default())
.await
.unwrap();
let mut count = 0; let mut count = 0;
while let Some(value) = stream.next().await { while let Some(value) = stream.next().await {
@ -334,17 +346,15 @@ mod tests {
Ok(v) => { Ok(v) => {
assert_eq!("a", v.attribute); assert_eq!("a", v.attribute);
} }
Err(e) => { Err(e) => match e {
match e { EvalError::Global(e) => {
EvalError::Global(e) => { let message = format!("{}", e);
let message = format!("{}", e); assert!(message.find("No such file or directory").is_some());
assert!(message.find("No such file or directory").is_some());
}
_ => {
panic!("Expected a global error, got {:?}", e);
}
} }
} _ => {
panic!("Expected a global error, got {:?}", e);
}
},
} }
count += 1; count += 1;
} }

View file

@ -7,7 +7,7 @@ use std::process::Stdio;
use serde::Deserialize; use serde::Deserialize;
use tokio::process::Command; use tokio::process::Command;
use super::{NixCheck, ColmenaError, ColmenaResult}; use super::{ColmenaError, ColmenaResult, NixCheck};
/// A Nix Flake. /// A Nix Flake.
#[derive(Debug)] #[derive(Debug)]
@ -27,7 +27,10 @@ impl Flake {
pub async fn from_dir<P: AsRef<Path>>(dir: P) -> ColmenaResult<Self> { pub async fn from_dir<P: AsRef<Path>>(dir: P) -> ColmenaResult<Self> {
NixCheck::require_flake_support().await?; NixCheck::require_flake_support().await?;
let flake = dir.as_ref().as_os_str().to_str() let flake = dir
.as_ref()
.as_os_str()
.to_str()
.expect("Flake directory path contains non-UTF-8 characters"); .expect("Flake directory path contains non-UTF-8 characters");
let info = FlakeMetadata::resolve(flake).await?; let info = FlakeMetadata::resolve(flake).await?;
@ -83,10 +86,9 @@ impl FlakeMetadata {
return Err(output.status.into()); return Err(output.status.into());
} }
serde_json::from_slice::<FlakeMetadata>(&output.stdout) serde_json::from_slice::<FlakeMetadata>(&output.stdout).map_err(|_| {
.map_err(|_| { let output = String::from_utf8_lossy(&output.stdout).to_string();
let output = String::from_utf8_lossy(&output.stdout).to_string(); ColmenaError::BadOutput { output }
ColmenaError::BadOutput { output } })
})
} }
} }

View file

@ -32,9 +32,7 @@ impl Assets {
create_file(&temp_dir, "options.nix", false, OPTIONS_NIX); create_file(&temp_dir, "options.nix", false, OPTIONS_NIX);
create_file(&temp_dir, "modules.nix", false, MODULES_NIX); create_file(&temp_dir, "modules.nix", false, MODULES_NIX);
Self { Self { temp_dir }
temp_dir,
}
} }
/// Returns the base expression from which the evaluated Hive can be used. /// Returns the base expression from which the evaluated Hive can be used.
@ -62,8 +60,12 @@ impl Assets {
} }
fn get_path(&self, name: &str) -> String { fn get_path(&self, name: &str) -> String {
self.temp_dir.path().join(name) self.temp_dir
.to_str().unwrap().to_string() .path()
.join(name)
.to_str()
.unwrap()
.to_string()
} }
} }

View file

@ -4,30 +4,24 @@ mod assets;
mod tests; mod tests;
use std::collections::HashMap; use std::collections::HashMap;
use std::convert::AsRef;
use std::io::Write; use std::io::Write;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::convert::AsRef;
use serde::Serialize;
use tempfile::{NamedTempFile, TempPath}; use tempfile::{NamedTempFile, TempPath};
use tokio::process::Command; use tokio::process::Command;
use tokio::sync::OnceCell; use tokio::sync::OnceCell;
use serde::Serialize;
use validator::Validate; use validator::Validate;
use super::{
Flake,
NixOptions,
NodeName,
NodeConfig,
NodeFilter,
NixExpression,
ProfileDerivation,
StorePath, MetaConfig,
};
use super::deployment::TargetNode; use super::deployment::TargetNode;
use super::{
Flake, MetaConfig, NixExpression, NixOptions, NodeConfig, NodeFilter, NodeName,
ProfileDerivation, StorePath,
};
use crate::error::ColmenaResult; use crate::error::ColmenaResult;
use crate::util::{CommandExecution, CommandExt};
use crate::job::JobHandle; use crate::job::JobHandle;
use crate::util::{CommandExecution, CommandExt};
use assets::Assets; use assets::Assets;
#[derive(Debug)] #[derive(Debug)]
@ -98,12 +92,8 @@ impl HivePath {
fn context_dir(&self) -> Option<PathBuf> { fn context_dir(&self) -> Option<PathBuf> {
match self { match self {
Self::Legacy(p) => { Self::Legacy(p) => p.parent().map(|d| d.to_owned()),
p.parent().map(|d| d.to_owned()) Self::Flake(flake) => flake.local_dir().map(|d| d.to_owned()),
}
Self::Flake(flake) => {
flake.local_dir().map(|d| d.to_owned())
}
} }
} }
} }
@ -112,7 +102,7 @@ impl Hive {
pub fn new(path: HivePath) -> ColmenaResult<Self> { pub fn new(path: HivePath) -> ColmenaResult<Self> {
let context_dir = path.context_dir(); let context_dir = path.context_dir();
Ok(Self{ Ok(Self {
path, path,
context_dir, context_dir,
assets: Assets::new(), assets: Assets::new(),
@ -126,10 +116,14 @@ impl Hive {
} }
pub async fn get_meta_config(&self) -> ColmenaResult<&MetaConfig> { pub async fn get_meta_config(&self) -> ColmenaResult<&MetaConfig> {
self.meta_config.get_or_try_init(||async { self.meta_config
self.nix_instantiate("hive.metaConfig").eval() .get_or_try_init(|| async {
.capture_json().await self.nix_instantiate("hive.metaConfig")
}).await .eval()
.capture_json()
.await
})
.await
} }
pub fn set_show_trace(&mut self, value: bool) { pub fn set_show_trace(&mut self, value: bool) {
@ -156,7 +150,12 @@ impl Hive {
} }
/// Convenience wrapper to filter nodes for CLI actions. /// Convenience wrapper to filter nodes for CLI actions.
pub async fn select_nodes(&self, filter: Option<NodeFilter>, ssh_config: Option<PathBuf>, ssh_only: bool) -> ColmenaResult<HashMap<NodeName, TargetNode>> { pub async fn select_nodes(
&self,
filter: Option<NodeFilter>,
ssh_config: Option<PathBuf>,
ssh_only: bool,
) -> ColmenaResult<HashMap<NodeName, TargetNode>> {
let mut node_configs = None; let mut node_configs = None;
log::info!("Enumerating nodes..."); log::info!("Enumerating nodes...");
@ -168,15 +167,16 @@ impl Hive {
log::debug!("Retrieving deployment info for all nodes..."); log::debug!("Retrieving deployment info for all nodes...");
let all_node_configs = self.deployment_info().await?; let all_node_configs = self.deployment_info().await?;
let filtered = filter.filter_node_configs(all_node_configs.iter()) let filtered = filter
.into_iter().collect(); .filter_node_configs(all_node_configs.iter())
.into_iter()
.collect();
node_configs = Some(all_node_configs); node_configs = Some(all_node_configs);
filtered filtered
} else { } else {
filter.filter_node_names(&all_nodes)? filter.filter_node_names(&all_nodes)?.into_iter().collect()
.into_iter().collect()
} }
} }
None => all_nodes.clone(), None => all_nodes.clone(),
@ -223,9 +223,18 @@ impl Hive {
} else if targets.len() == all_nodes.len() { } else if targets.len() == all_nodes.len() {
log::info!("Selected all {} nodes.", targets.len()); log::info!("Selected all {} nodes.", targets.len());
} else if !ssh_only || skipped == 0 { } else if !ssh_only || skipped == 0 {
log::info!("Selected {} out of {} hosts.", targets.len(), all_nodes.len()); log::info!(
"Selected {} out of {} hosts.",
targets.len(),
all_nodes.len()
);
} else { } else {
log::info!("Selected {} out of {} hosts ({} skipped).", targets.len(), all_nodes.len(), skipped); log::info!(
"Selected {} out of {} hosts ({} skipped).",
targets.len(),
all_nodes.len(),
skipped
);
} }
Ok(targets) Ok(targets)
@ -233,14 +242,20 @@ impl Hive {
/// Returns a list of all node names. /// Returns a list of all node names.
pub async fn node_names(&self) -> ColmenaResult<Vec<NodeName>> { pub async fn node_names(&self) -> ColmenaResult<Vec<NodeName>> {
self.nix_instantiate("attrNames hive.nodes").eval() self.nix_instantiate("attrNames hive.nodes")
.capture_json().await .eval()
.capture_json()
.await
} }
/// Retrieve deployment info for all nodes. /// Retrieve deployment info for all nodes.
pub async fn deployment_info(&self) -> ColmenaResult<HashMap<NodeName, NodeConfig>> { pub async fn deployment_info(&self) -> ColmenaResult<HashMap<NodeName, NodeConfig>> {
let configs: HashMap<NodeName, NodeConfig> = self.nix_instantiate("hive.deploymentConfig").eval_with_builders().await? let configs: HashMap<NodeName, NodeConfig> = self
.capture_json().await?; .nix_instantiate("hive.deploymentConfig")
.eval_with_builders()
.await?
.capture_json()
.await?;
for config in configs.values() { for config in configs.values() {
config.validate()?; config.validate()?;
@ -253,19 +268,34 @@ impl Hive {
/// Retrieve deployment info for a single node. /// Retrieve deployment info for a single node.
#[cfg_attr(not(target_os = "linux"), allow(dead_code))] #[cfg_attr(not(target_os = "linux"), allow(dead_code))]
pub async fn deployment_info_single(&self, node: &NodeName) -> ColmenaResult<Option<NodeConfig>> { pub async fn deployment_info_single(
&self,
node: &NodeName,
) -> ColmenaResult<Option<NodeConfig>> {
let expr = format!("hive.nodes.\"{}\".config.deployment or null", node.as_str()); let expr = format!("hive.nodes.\"{}\".config.deployment or null", node.as_str());
self.nix_instantiate(&expr).eval_with_builders().await? self.nix_instantiate(&expr)
.capture_json().await .eval_with_builders()
.await?
.capture_json()
.await
} }
/// Retrieve deployment info for a list of nodes. /// Retrieve deployment info for a list of nodes.
pub async fn deployment_info_selected(&self, nodes: &[NodeName]) -> ColmenaResult<HashMap<NodeName, NodeConfig>> { pub async fn deployment_info_selected(
&self,
nodes: &[NodeName],
) -> ColmenaResult<HashMap<NodeName, NodeConfig>> {
let nodes_expr = SerializedNixExpression::new(nodes)?; let nodes_expr = SerializedNixExpression::new(nodes)?;
let configs: HashMap<NodeName, NodeConfig> = self.nix_instantiate(&format!("hive.deploymentConfigSelected {}", nodes_expr.expression())) let configs: HashMap<NodeName, NodeConfig> = self
.eval_with_builders().await? .nix_instantiate(&format!(
.capture_json().await?; "hive.deploymentConfigSelected {}",
nodes_expr.expression()
))
.eval_with_builders()
.await?
.capture_json()
.await?;
for config in configs.values() { for config in configs.values() {
config.validate()?; config.validate()?;
@ -282,20 +312,25 @@ impl Hive {
/// Evaluation may take up a lot of memory, so we make it possible /// Evaluation may take up a lot of memory, so we make it possible
/// to split up the evaluation process into chunks and run them /// to split up the evaluation process into chunks and run them
/// concurrently with other processes (e.g., build and apply). /// concurrently with other processes (e.g., build and apply).
pub async fn eval_selected(&self, nodes: &[NodeName], job: Option<JobHandle>) -> ColmenaResult<HashMap<NodeName, ProfileDerivation>> { pub async fn eval_selected(
&self,
nodes: &[NodeName],
job: Option<JobHandle>,
) -> ColmenaResult<HashMap<NodeName, ProfileDerivation>> {
let nodes_expr = SerializedNixExpression::new(nodes)?; let nodes_expr = SerializedNixExpression::new(nodes)?;
let expr = format!("hive.evalSelectedDrvPaths {}", nodes_expr.expression()); let expr = format!("hive.evalSelectedDrvPaths {}", nodes_expr.expression());
let command = self.nix_instantiate(&expr) let command = self.nix_instantiate(&expr).eval_with_builders().await?;
.eval_with_builders().await?;
let mut execution = CommandExecution::new(command); let mut execution = CommandExecution::new(command);
execution.set_job(job); execution.set_job(job);
execution.set_hide_stdout(true); execution.set_hide_stdout(true);
execution execution
.capture_json::<HashMap<NodeName, StorePath>>().await? .capture_json::<HashMap<NodeName, StorePath>>()
.into_iter().map(|(name, path)| { .await?
.into_iter()
.map(|(name, path)| {
let path = path.into_derivation()?; let path = path.into_derivation()?;
Ok((name, path)) Ok((name, path))
}) })
@ -316,12 +351,18 @@ impl Hive {
pub async fn introspect(&self, expression: String, instantiate: bool) -> ColmenaResult<String> { pub async fn introspect(&self, expression: String, instantiate: bool) -> ColmenaResult<String> {
if instantiate { if instantiate {
let expression = format!("hive.introspect ({})", expression); let expression = format!("hive.introspect ({})", expression);
self.nix_instantiate(&expression).instantiate_with_builders().await? self.nix_instantiate(&expression)
.capture_output().await .instantiate_with_builders()
.await?
.capture_output()
.await
} else { } else {
let expression = format!("toJSON (hive.introspect ({}))", expression); let expression = format!("toJSON (hive.introspect ({}))", expression);
self.nix_instantiate(&expression).eval_with_builders().await? self.nix_instantiate(&expression)
.capture_json().await .eval_with_builders()
.await?
.capture_json()
.await
} }
} }
@ -346,10 +387,7 @@ impl Hive {
impl<'hive> NixInstantiate<'hive> { impl<'hive> NixInstantiate<'hive> {
fn new(hive: &'hive Hive, expression: String) -> Self { fn new(hive: &'hive Hive, expression: String) -> Self {
Self { Self { hive, expression }
hive,
expression,
}
} }
fn instantiate(&self) -> Command { fn instantiate(&self) -> Command {
@ -373,7 +411,10 @@ impl<'hive> NixInstantiate<'hive> {
fn eval(self) -> Command { fn eval(self) -> Command {
let mut command = self.instantiate(); let mut command = self.instantiate();
let options = self.hive.nix_options(); let options = self.hive.nix_options();
command.arg("--eval").arg("--json").arg("--strict") command
.arg("--eval")
.arg("--json")
.arg("--strict")
// Ensures the derivations are instantiated // Ensures the derivations are instantiated
// Required for system profile evaluation and IFD // Required for system profile evaluation and IFD
.arg("--read-write-mode") .arg("--read-write-mode")
@ -401,7 +442,10 @@ impl<'hive> NixInstantiate<'hive> {
} }
impl SerializedNixExpression { impl SerializedNixExpression {
pub fn new<T>(data: T) -> ColmenaResult<Self> where T: Serialize { pub fn new<T>(data: T) -> ColmenaResult<Self>
where
T: Serialize,
{
let mut tmp = NamedTempFile::new()?; let mut tmp = NamedTempFile::new()?;
let json = serde_json::to_vec(&data).expect("Could not serialize data"); let json = serde_json::to_vec(&data).expect("Could not serialize data");
tmp.write_all(&json)?; tmp.write_all(&json)?;
@ -414,7 +458,10 @@ impl SerializedNixExpression {
impl NixExpression for SerializedNixExpression { impl NixExpression for SerializedNixExpression {
fn expression(&self) -> String { fn expression(&self) -> String {
format!("(builtins.fromJSON (builtins.readFile {}))", self.json_file.to_str().unwrap()) format!(
"(builtins.fromJSON (builtins.readFile {}))",
self.json_file.to_str().unwrap()
)
} }
} }

View file

@ -15,7 +15,7 @@ use tokio_test::block_on;
macro_rules! node { macro_rules! node {
($n:expr) => { ($n:expr) => {
NodeName::new($n.to_string()).unwrap() NodeName::new($n.to_string()).unwrap()
} };
} }
fn set_eq<T>(a: &[T], b: &[T]) -> bool fn set_eq<T>(a: &[T], b: &[T]) -> bool
@ -92,7 +92,8 @@ impl Deref for TempHive {
#[test] #[test]
fn test_parse_simple() { fn test_parse_simple() {
let hive = TempHive::new(r#" let hive = TempHive::new(
r#"
{ {
defaults = { pkgs, ... }: { defaults = { pkgs, ... }: {
environment.systemPackages = with pkgs; [ environment.systemPackages = with pkgs; [
@ -123,7 +124,8 @@ fn test_parse_simple() {
time.timeZone = "America/Los_Angeles"; time.timeZone = "America/Los_Angeles";
}; };
} }
"#); "#,
);
let nodes = block_on(hive.deployment_info()).unwrap(); let nodes = block_on(hive.deployment_info()).unwrap();
assert!(set_eq( assert!(set_eq(
@ -135,7 +137,11 @@ fn test_parse_simple() {
let host_a = &nodes[&node!("host-a")]; let host_a = &nodes[&node!("host-a")];
assert!(set_eq( assert!(set_eq(
&["common-tag", "a-tag"], &["common-tag", "a-tag"],
&host_a.tags.iter().map(String::as_str).collect::<Vec<&str>>(), &host_a
.tags
.iter()
.map(String::as_str)
.collect::<Vec<&str>>(),
)); ));
assert_eq!(Some("host-a"), host_a.target_host.as_deref()); assert_eq!(Some("host-a"), host_a.target_host.as_deref());
assert_eq!(None, host_a.target_port); assert_eq!(None, host_a.target_port);
@ -145,7 +151,11 @@ fn test_parse_simple() {
let host_b = &nodes[&node!("host-b")]; let host_b = &nodes[&node!("host-b")];
assert!(set_eq( assert!(set_eq(
&["common-tag"], &["common-tag"],
&host_b.tags.iter().map(String::as_str).collect::<Vec<&str>>(), &host_b
.tags
.iter()
.map(String::as_str)
.collect::<Vec<&str>>(),
)); ));
assert_eq!(Some("somehost.tld"), host_b.target_host.as_deref()); assert_eq!(Some("somehost.tld"), host_b.target_host.as_deref());
assert_eq!(Some(1234), host_b.target_port); assert_eq!(Some(1234), host_b.target_port);
@ -171,7 +181,8 @@ fn test_parse_flake() {
#[test] #[test]
fn test_parse_node_references() { fn test_parse_node_references() {
TempHive::valid(r#" TempHive::valid(
r#"
with builtins; with builtins;
{ {
host-a = { name, nodes, ... }: host-a = { name, nodes, ... }:
@ -186,23 +197,27 @@ fn test_parse_node_references() {
assert nodes.host-a.config.time.timeZone == "America/Los_Angeles"; assert nodes.host-a.config.time.timeZone == "America/Los_Angeles";
{}; {};
} }
"#); "#,
);
} }
#[test] #[test]
fn test_parse_unknown_option() { fn test_parse_unknown_option() {
TempHive::invalid(r#" TempHive::invalid(
r#"
{ {
bad = { bad = {
deployment.noSuchOption = "not kidding"; deployment.noSuchOption = "not kidding";
}; };
} }
"#); "#,
);
} }
#[test] #[test]
fn test_config_list() { fn test_config_list() {
TempHive::valid(r#" TempHive::valid(
r#"
with builtins; with builtins;
{ {
host-a = [ host-a = [
@ -219,12 +234,14 @@ fn test_config_list() {
assert elem "some-tag" nodes.host-a.config.deployment.tags; assert elem "some-tag" nodes.host-a.config.deployment.tags;
{}; {};
} }
"#); "#,
);
} }
#[test] #[test]
fn test_parse_key_text() { fn test_parse_key_text() {
TempHive::valid(r#" TempHive::valid(
r#"
{ {
test = { test = {
deployment.keys.topSecret = { deployment.keys.topSecret = {
@ -232,12 +249,14 @@ fn test_parse_key_text() {
}; };
}; };
} }
"#); "#,
);
} }
#[test] #[test]
fn test_parse_key_command_good() { fn test_parse_key_command_good() {
TempHive::valid(r#" TempHive::valid(
r#"
{ {
test = { test = {
deployment.keys.elohim = { deployment.keys.elohim = {
@ -245,12 +264,14 @@ fn test_parse_key_command_good() {
}; };
}; };
} }
"#); "#,
);
} }
#[test] #[test]
fn test_parse_key_command_bad() { fn test_parse_key_command_bad() {
TempHive::invalid(r#" TempHive::invalid(
r#"
{ {
test = { test = {
deployment.keys.elohim = { deployment.keys.elohim = {
@ -258,12 +279,14 @@ fn test_parse_key_command_bad() {
}; };
}; };
} }
"#); "#,
);
} }
#[test] #[test]
fn test_parse_key_file() { fn test_parse_key_file() {
TempHive::valid(r#" TempHive::valid(
r#"
{ {
test = { test = {
deployment.keys.l337hax0rwow = { deployment.keys.l337hax0rwow = {
@ -271,27 +294,32 @@ fn test_parse_key_file() {
}; };
}; };
} }
"#); "#,
);
} }
#[test] #[test]
fn test_eval_non_existent_pkg() { fn test_eval_non_existent_pkg() {
// Sanity check // Sanity check
TempHive::eval_failure(r#" TempHive::eval_failure(
r#"
{ {
test = { pkgs, ... }: { test = { pkgs, ... }: {
boot.isContainer = true; boot.isContainer = true;
environment.systemPackages = with pkgs; [ thisPackageDoesNotExist ]; environment.systemPackages = with pkgs; [ thisPackageDoesNotExist ];
}; };
} }
"#, vec![ node!("test") ]); "#,
vec![node!("test")],
);
} }
// Nixpkgs config tests // Nixpkgs config tests
#[test] #[test]
fn test_nixpkgs_system() { fn test_nixpkgs_system() {
TempHive::valid(r#" TempHive::valid(
r#"
{ {
meta = { meta = {
nixpkgs = import <nixpkgs> { nixpkgs = import <nixpkgs> {
@ -302,9 +330,11 @@ fn test_nixpkgs_system() {
boot.isContainer = assert pkgs.system == "armv5tel-linux"; true; boot.isContainer = assert pkgs.system == "armv5tel-linux"; true;
}; };
} }
"#); "#,
);
TempHive::valid(r#" TempHive::valid(
r#"
{ {
meta = { meta = {
nixpkgs = import <nixpkgs> { nixpkgs = import <nixpkgs> {
@ -316,12 +346,14 @@ fn test_nixpkgs_system() {
boot.isContainer = assert pkgs.system == "armv5tel-linux"; true; boot.isContainer = assert pkgs.system == "armv5tel-linux"; true;
}; };
} }
"#); "#,
);
} }
#[test] #[test]
fn test_nixpkgs_path_like() { fn test_nixpkgs_path_like() {
TempHive::valid(r#" TempHive::valid(
r#"
{ {
meta = { meta = {
nixpkgs = { nixpkgs = {
@ -332,13 +364,15 @@ fn test_nixpkgs_path_like() {
boot.isContainer = true; boot.isContainer = true;
}; };
} }
"#); "#,
);
} }
#[test] #[test]
fn test_nixpkgs_overlay_meta_nixpkgs() { fn test_nixpkgs_overlay_meta_nixpkgs() {
// Only set overlays in meta.nixpkgs // Only set overlays in meta.nixpkgs
TempHive::eval_success(r#" TempHive::eval_success(
r#"
{ {
meta = { meta = {
nixpkgs = import <nixpkgs> { nixpkgs = import <nixpkgs> {
@ -352,13 +386,16 @@ fn test_nixpkgs_overlay_meta_nixpkgs() {
environment.systemPackages = with pkgs; [ my-coreutils ]; environment.systemPackages = with pkgs; [ my-coreutils ];
}; };
} }
"#, vec![ node!("test") ]); "#,
vec![node!("test")],
);
} }
#[test] #[test]
fn test_nixpkgs_overlay_node_config() { fn test_nixpkgs_overlay_node_config() {
// Only set overlays in node config // Only set overlays in node config
TempHive::eval_success(r#" TempHive::eval_success(
r#"
{ {
test = { pkgs, ... }: { test = { pkgs, ... }: {
boot.isContainer = true; boot.isContainer = true;
@ -368,13 +405,16 @@ fn test_nixpkgs_overlay_node_config() {
environment.systemPackages = with pkgs; [ my-coreutils ]; environment.systemPackages = with pkgs; [ my-coreutils ];
}; };
} }
"#, vec![ node!("test") ]); "#,
vec![node!("test")],
);
} }
#[test] #[test]
fn test_nixpkgs_overlay_both() { fn test_nixpkgs_overlay_both() {
// Set overlays both in meta.nixpkgs and in node config // Set overlays both in meta.nixpkgs and in node config
TempHive::eval_success(r#" TempHive::eval_success(
r#"
{ {
meta = { meta = {
nixpkgs = import <nixpkgs> { nixpkgs = import <nixpkgs> {
@ -391,13 +431,16 @@ fn test_nixpkgs_overlay_both() {
environment.systemPackages = with pkgs; [ meta-coreutils node-busybox ]; environment.systemPackages = with pkgs; [ meta-coreutils node-busybox ];
}; };
} }
"#, vec![ node!("test") ]); "#,
vec![node!("test")],
);
} }
#[test] #[test]
fn test_nixpkgs_config_meta_nixpkgs() { fn test_nixpkgs_config_meta_nixpkgs() {
// Set config in meta.nixpkgs // Set config in meta.nixpkgs
TempHive::eval_success(r#" TempHive::eval_success(
r#"
{ {
meta = { meta = {
nixpkgs = import <nixpkgs> { nixpkgs = import <nixpkgs> {
@ -413,13 +456,16 @@ fn test_nixpkgs_config_meta_nixpkgs() {
boot.isContainer = assert pkgs.config.allowUnfree; true; boot.isContainer = assert pkgs.config.allowUnfree; true;
}; };
} }
"#, vec![ node!("test") ]); "#,
vec![node!("test")],
);
} }
#[test] #[test]
fn test_nixpkgs_config_node_config() { fn test_nixpkgs_config_node_config() {
// Set config in node config // Set config in node config
TempHive::eval_success(r#" TempHive::eval_success(
r#"
{ {
test = { pkgs, ... }: { test = { pkgs, ... }: {
nixpkgs.config = { nixpkgs.config = {
@ -428,7 +474,9 @@ fn test_nixpkgs_config_node_config() {
boot.isContainer = assert pkgs.config.allowUnfree; true; boot.isContainer = assert pkgs.config.allowUnfree; true;
}; };
} }
"#, vec![ node!("test") ]); "#,
vec![node!("test")],
);
} }
#[test] #[test]
@ -457,7 +505,7 @@ fn test_nixpkgs_config_override() {
.replace("META_VAL", "true") .replace("META_VAL", "true")
.replace("NODE_VAL", "false") .replace("NODE_VAL", "false")
.replace("EXPECTED_VAL", "false"), .replace("EXPECTED_VAL", "false"),
vec![ node!("test") ] vec![node!("test")],
); );
TempHive::eval_success( TempHive::eval_success(
@ -465,13 +513,14 @@ fn test_nixpkgs_config_override() {
.replace("META_VAL", "false") .replace("META_VAL", "false")
.replace("NODE_VAL", "true") .replace("NODE_VAL", "true")
.replace("EXPECTED_VAL", "true"), .replace("EXPECTED_VAL", "true"),
vec![ node!("test") ] vec![node!("test")],
); );
} }
#[test] #[test]
fn test_meta_special_args() { fn test_meta_special_args() {
TempHive::valid(r#" TempHive::valid(
r#"
{ {
meta.specialArgs = { meta.specialArgs = {
undine = "assimilated"; undine = "assimilated";
@ -483,12 +532,14 @@ fn test_meta_special_args() {
boot.isContainer = true; boot.isContainer = true;
}; };
} }
"#); "#,
);
} }
#[test] #[test]
fn test_meta_node_special_args() { fn test_meta_node_special_args() {
TempHive::valid(r#" TempHive::valid(
r#"
{ {
meta.specialArgs = { meta.specialArgs = {
someArg = "global"; someArg = "global";
@ -510,12 +561,14 @@ fn test_meta_node_special_args() {
boot.isContainer = true; boot.isContainer = true;
}; };
} }
"#); "#,
);
} }
#[test] #[test]
fn test_hive_autocall() { fn test_hive_autocall() {
TempHive::valid(r#" TempHive::valid(
r#"
{ {
argument ? "with default value" argument ? "with default value"
}: { }: {
@ -523,9 +576,11 @@ fn test_hive_autocall() {
boot.isContainer = true; boot.isContainer = true;
}; };
} }
"#); "#,
);
TempHive::valid(r#" TempHive::valid(
r#"
{ {
some = "value"; some = "value";
__functor = self: { argument ? "with default value" }: { __functor = self: { argument ? "with default value" }: {
@ -534,9 +589,11 @@ fn test_hive_autocall() {
}; };
}; };
} }
"#); "#,
);
TempHive::invalid(r#" TempHive::invalid(
r#"
{ {
thisWontWork thisWontWork
}: { }: {
@ -544,47 +601,51 @@ fn test_hive_autocall() {
boot.isContainer = true; boot.isContainer = true;
}; };
} }
"#); "#,
);
} }
#[test] #[test]
fn test_hive_introspect() { fn test_hive_introspect() {
let hive = TempHive::new(r#" let hive = TempHive::new(
r#"
{ {
test = { ... }: { test = { ... }: {
boot.isContainer = true; boot.isContainer = true;
}; };
} }
"#); "#,
);
let expr = r#" let expr = r#"
{ pkgs, lib, nodes }: { pkgs, lib, nodes }:
assert pkgs ? hello; assert pkgs ? hello;
assert lib ? versionAtLeast; assert lib ? versionAtLeast;
nodes.test.config.boot.isContainer nodes.test.config.boot.isContainer
"#.to_string(); "#
.to_string();
let eval = block_on(hive.introspect(expr, false)) let eval = block_on(hive.introspect(expr, false)).unwrap();
.unwrap();
assert_eq!("true", eval); assert_eq!("true", eval);
} }
#[test] #[test]
fn test_hive_get_meta() { fn test_hive_get_meta() {
let hive = TempHive::new(r#" let hive = TempHive::new(
r#"
{ {
meta.allowApplyAll = false; meta.allowApplyAll = false;
meta.specialArgs = { meta.specialArgs = {
this_is_new = false; this_is_new = false;
}; };
} }
"#); "#,
);
let eval = block_on(hive.get_meta_config()) let eval = block_on(hive.get_meta_config()).unwrap();
.unwrap();
eprintln!("{:?}", eval); eprintln!("{:?}", eval);
assert!(!eval.allow_apply_all); assert!(!eval.allow_apply_all);
} }

View file

@ -19,24 +19,36 @@ use crate::util::capture_stream;
const SCRIPT_TEMPLATE: &str = include_str!("./key_uploader.template.sh"); const SCRIPT_TEMPLATE: &str = include_str!("./key_uploader.template.sh");
pub fn generate_script<'a>(key: &'a Key, destination: &'a Path, require_ownership: bool) -> Cow<'a, str> { pub fn generate_script<'a>(
let key_script = SCRIPT_TEMPLATE.to_string() key: &'a Key,
destination: &'a Path,
require_ownership: bool,
) -> Cow<'a, str> {
let key_script = SCRIPT_TEMPLATE
.to_string()
.replace("%DESTINATION%", destination.to_str().unwrap()) .replace("%DESTINATION%", destination.to_str().unwrap())
.replace("%USER%", &escape(key.user().into())) .replace("%USER%", &escape(key.user().into()))
.replace("%GROUP%", &escape(key.group().into())) .replace("%GROUP%", &escape(key.group().into()))
.replace("%PERMISSIONS%", &escape(key.permissions().into())) .replace("%PERMISSIONS%", &escape(key.permissions().into()))
.replace("%REQUIRE_OWNERSHIP%", if require_ownership { "1" } else { "" }) .replace(
.trim_end_matches('\n').to_string(); "%REQUIRE_OWNERSHIP%",
if require_ownership { "1" } else { "" },
)
.trim_end_matches('\n')
.to_string();
escape(key_script.into()) escape(key_script.into())
} }
pub async fn feed_uploader(mut uploader: Child, key: &Key, job: Option<JobHandle>) -> ColmenaResult<()> { pub async fn feed_uploader(
let mut reader = key.reader().await mut uploader: Child,
.map_err(|error| ColmenaError::KeyError { key: &Key,
name: key.name().to_owned(), job: Option<JobHandle>,
error, ) -> ColmenaResult<()> {
})?; let mut reader = key.reader().await.map_err(|error| ColmenaError::KeyError {
name: key.name().to_owned(),
error,
})?;
let mut stdin = uploader.stdin.take().unwrap(); let mut stdin = uploader.stdin.take().unwrap();
tokio::io::copy(reader.as_mut(), &mut stdin).await?; tokio::io::copy(reader.as_mut(), &mut stdin).await?;
@ -52,7 +64,8 @@ pub async fn feed_uploader(mut uploader: Child, key: &Key, job: Option<JobHandle
uploader.wait(), uploader.wait(),
); );
let (stdout, stderr, exit) = futures.await; let (stdout, stderr, exit) = futures.await;
stdout?; stderr?; stdout?;
stderr?;
let exit = exit?; let exit = exit?;

View file

@ -1,15 +1,15 @@
use std::convert::TryInto;
use std::collections::HashMap; use std::collections::HashMap;
use std::convert::TryInto;
use std::process::Stdio; use std::process::Stdio;
use async_trait::async_trait; use async_trait::async_trait;
use tokio::process::Command; use tokio::process::Command;
use crate::error::{ColmenaResult, ColmenaError}; use super::{key_uploader, CopyDirection, CopyOptions, Host};
use crate::nix::{StorePath, Profile, Goal, Key, NixOptions, SYSTEM_PROFILE, CURRENT_PROFILE}; use crate::error::{ColmenaError, ColmenaResult};
use crate::util::{CommandExecution, CommandExt};
use crate::job::JobHandle; use crate::job::JobHandle;
use super::{CopyDirection, CopyOptions, Host, key_uploader}; use crate::nix::{Goal, Key, NixOptions, Profile, StorePath, CURRENT_PROFILE, SYSTEM_PROFILE};
use crate::util::{CommandExecution, CommandExt};
/// The local machine running Colmena. /// The local machine running Colmena.
/// ///
@ -34,7 +34,12 @@ impl Local {
#[async_trait] #[async_trait]
impl Host for Local { impl Host for Local {
async fn copy_closure(&mut self, _closure: &StorePath, _direction: CopyDirection, _options: CopyOptions) -> ColmenaResult<()> { async fn copy_closure(
&mut self,
_closure: &StorePath,
_direction: CopyDirection,
_options: CopyOptions,
) -> ColmenaResult<()> {
Ok(()) Ok(())
} }
@ -54,11 +59,18 @@ impl Host for Local {
execution.run().await?; execution.run().await?;
let (stdout, _) = execution.get_logs(); let (stdout, _) = execution.get_logs();
stdout.unwrap().lines() stdout
.map(|p| p.to_string().try_into()).collect() .unwrap()
.lines()
.map(|p| p.to_string().try_into())
.collect()
} }
async fn upload_keys(&mut self, keys: &HashMap<String, Key>, require_ownership: bool) -> ColmenaResult<()> { async fn upload_keys(
&mut self,
keys: &HashMap<String, Key>,
require_ownership: bool,
) -> ColmenaResult<()> {
for (name, key) in keys { for (name, key) in keys {
self.upload_key(name, key, require_ownership).await?; self.upload_key(name, key, require_ownership).await?;
} }
@ -98,7 +110,10 @@ impl Host for Local {
.capture_output() .capture_output()
.await?; .await?;
let path = paths.lines().into_iter().next() let path = paths
.lines()
.into_iter()
.next()
.ok_or(ColmenaError::FailedToGetCurrentProfile)? .ok_or(ColmenaError::FailedToGetCurrentProfile)?
.to_string() .to_string()
.try_into()?; .try_into()?;
@ -108,11 +123,20 @@ impl Host for Local {
async fn get_main_system_profile(&mut self) -> ColmenaResult<Profile> { async fn get_main_system_profile(&mut self) -> ColmenaResult<Profile> {
let paths = Command::new("sh") let paths = Command::new("sh")
.args(&["-c", &format!("readlink -e {} || readlink -e {}", SYSTEM_PROFILE, CURRENT_PROFILE)]) .args(&[
"-c",
&format!(
"readlink -e {} || readlink -e {}",
SYSTEM_PROFILE, CURRENT_PROFILE
),
])
.capture_output() .capture_output()
.await?; .await?;
let path = paths.lines().into_iter().next() let path = paths
.lines()
.into_iter()
.next()
.ok_or(ColmenaError::FailedToGetCurrentProfile)? .ok_or(ColmenaError::FailedToGetCurrentProfile)?
.to_string() .to_string()
.try_into()?; .try_into()?;
@ -135,13 +159,21 @@ impl Local {
} }
/// "Uploads" a single key. /// "Uploads" a single key.
async fn upload_key(&mut self, name: &str, key: &Key, require_ownership: bool) -> ColmenaResult<()> { async fn upload_key(
&mut self,
name: &str,
key: &Key,
require_ownership: bool,
) -> ColmenaResult<()> {
if let Some(job) = &self.job { if let Some(job) = &self.job {
job.message(format!("Deploying key {}", name))?; job.message(format!("Deploying key {}", name))?;
} }
let path = key.path(); let path = key.path();
let key_script = format!("'{}'", key_uploader::generate_script(key, path, require_ownership)); let key_script = format!(
"'{}'",
key_uploader::generate_script(key, path, require_ownership)
);
let mut command = self.make_privileged_command(&["sh", "-c", &key_script]); let mut command = self.make_privileged_command(&["sh", "-c", &key_script]);
command.stdin(Stdio::piped()); command.stdin(Stdio::piped());

View file

@ -2,9 +2,9 @@ use std::collections::HashMap;
use async_trait::async_trait; use async_trait::async_trait;
use super::{Goal, Key, Profile, StorePath};
use crate::error::{ColmenaError, ColmenaResult}; use crate::error::{ColmenaError, ColmenaResult};
use crate::job::JobHandle; use crate::job::JobHandle;
use super::{StorePath, Profile, Goal, Key};
mod ssh; mod ssh;
pub use ssh::Ssh; pub use ssh::Ssh;
@ -92,7 +92,12 @@ pub trait Host: Send + Sync + std::fmt::Debug {
/// Sends or receives the specified closure to the host /// Sends or receives the specified closure to the host
/// ///
/// The StorePath and its dependent paths will then exist on this host. /// The StorePath and its dependent paths will then exist on this host.
async fn copy_closure(&mut self, closure: &StorePath, direction: CopyDirection, options: CopyOptions) -> ColmenaResult<()>; async fn copy_closure(
&mut self,
closure: &StorePath,
direction: CopyDirection,
options: CopyOptions,
) -> ColmenaResult<()>;
/// Realizes the specified derivation on the host /// Realizes the specified derivation on the host
/// ///
@ -106,19 +111,30 @@ pub trait Host: Send + Sync + std::fmt::Debug {
/// Realizes the specified local derivation on the host then retrieves the outputs. /// Realizes the specified local derivation on the host then retrieves the outputs.
async fn realize(&mut self, derivation: &StorePath) -> ColmenaResult<Vec<StorePath>> { async fn realize(&mut self, derivation: &StorePath) -> ColmenaResult<Vec<StorePath>> {
let options = CopyOptions::default() let options = CopyOptions::default().include_outputs(true);
.include_outputs(true);
self.copy_closure(derivation, CopyDirection::ToRemote, options).await?; self.copy_closure(derivation, CopyDirection::ToRemote, options)
.await?;
let paths = self.realize_remote(derivation).await?; let paths = self.realize_remote(derivation).await?;
self.copy_closure(derivation, CopyDirection::FromRemote, options).await?; self.copy_closure(derivation, CopyDirection::FromRemote, options)
.await?;
Ok(paths) Ok(paths)
} }
/// Pushes and optionally activates a profile to the host. /// Pushes and optionally activates a profile to the host.
async fn deploy(&mut self, profile: &Profile, goal: Goal, copy_options: CopyOptions) -> ColmenaResult<()> { async fn deploy(
self.copy_closure(profile.as_store_path(), CopyDirection::ToRemote, copy_options).await?; &mut self,
profile: &Profile,
goal: Goal,
copy_options: CopyOptions,
) -> ColmenaResult<()> {
self.copy_closure(
profile.as_store_path(),
CopyDirection::ToRemote,
copy_options,
)
.await?;
if goal.requires_activation() { if goal.requires_activation() {
self.activate(profile, goal).await?; self.activate(profile, goal).await?;
@ -133,7 +149,11 @@ pub trait Host: Send + Sync + std::fmt::Debug {
/// will not be applied if the specified user/group does not /// will not be applied if the specified user/group does not
/// exist. /// exist.
#[allow(unused_variables)] #[allow(unused_variables)]
async fn upload_keys(&mut self, keys: &HashMap<String, Key>, require_ownership: bool) -> ColmenaResult<()> { async fn upload_keys(
&mut self,
keys: &HashMap<String, Key>,
require_ownership: bool,
) -> ColmenaResult<()> {
Err(ColmenaError::Unsupported) Err(ColmenaError::Unsupported)
} }

View file

@ -8,11 +8,11 @@ use async_trait::async_trait;
use tokio::process::Command; use tokio::process::Command;
use tokio::time::sleep; use tokio::time::sleep;
use crate::error::{ColmenaResult, ColmenaError}; use super::{key_uploader, CopyDirection, CopyOptions, Host, RebootOptions};
use crate::nix::{StorePath, Profile, Goal, Key, SYSTEM_PROFILE, CURRENT_PROFILE}; use crate::error::{ColmenaError, ColmenaResult};
use crate::util::{CommandExecution, CommandExt};
use crate::job::JobHandle; use crate::job::JobHandle;
use super::{CopyDirection, CopyOptions, RebootOptions, Host, key_uploader}; use crate::nix::{Goal, Key, Profile, StorePath, CURRENT_PROFILE, SYSTEM_PROFILE};
use crate::util::{CommandExecution, CommandExt};
/// A remote machine connected over SSH. /// A remote machine connected over SSH.
#[derive(Debug)] #[derive(Debug)]
@ -41,20 +41,28 @@ struct BootId(String);
#[async_trait] #[async_trait]
impl Host for Ssh { impl Host for Ssh {
async fn copy_closure(&mut self, closure: &StorePath, direction: CopyDirection, options: CopyOptions) -> ColmenaResult<()> { async fn copy_closure(
&mut self,
closure: &StorePath,
direction: CopyDirection,
options: CopyOptions,
) -> ColmenaResult<()> {
let command = self.nix_copy_closure(closure, direction, options); let command = self.nix_copy_closure(closure, direction, options);
self.run_command(command).await self.run_command(command).await
} }
async fn realize_remote(&mut self, derivation: &StorePath) -> ColmenaResult<Vec<StorePath>> { async fn realize_remote(&mut self, derivation: &StorePath) -> ColmenaResult<Vec<StorePath>> {
let command = self.ssh(&["nix-store", "--no-gc-warning", "--realise", derivation.as_path().to_str().unwrap()]); let command = self.ssh(&[
"nix-store",
"--no-gc-warning",
"--realise",
derivation.as_path().to_str().unwrap(),
]);
let mut execution = CommandExecution::new(command); let mut execution = CommandExecution::new(command);
execution.set_job(self.job.clone()); execution.set_job(self.job.clone());
let paths = execution let paths = execution.capture_output().await?;
.capture_output()
.await?;
paths.lines().map(|p| p.to_string().try_into()).collect() paths.lines().map(|p| p.to_string().try_into()).collect()
} }
@ -63,7 +71,11 @@ impl Host for Ssh {
self.job = job; self.job = job;
} }
async fn upload_keys(&mut self, keys: &HashMap<String, Key>, require_ownership: bool) -> ColmenaResult<()> { async fn upload_keys(
&mut self,
keys: &HashMap<String, Key>,
require_ownership: bool,
) -> ColmenaResult<()> {
for (name, key) in keys { for (name, key) in keys {
self.upload_key(name, key, require_ownership).await?; self.upload_key(name, key, require_ownership).await?;
} }
@ -89,11 +101,15 @@ impl Host for Ssh {
} }
async fn get_current_system_profile(&mut self) -> ColmenaResult<Profile> { async fn get_current_system_profile(&mut self) -> ColmenaResult<Profile> {
let paths = self.ssh(&["readlink", "-e", CURRENT_PROFILE]) let paths = self
.ssh(&["readlink", "-e", CURRENT_PROFILE])
.capture_output() .capture_output()
.await?; .await?;
let path = paths.lines().into_iter().next() let path = paths
.lines()
.into_iter()
.next()
.ok_or(ColmenaError::FailedToGetCurrentProfile)? .ok_or(ColmenaError::FailedToGetCurrentProfile)?
.to_string() .to_string()
.try_into()?; .try_into()?;
@ -102,13 +118,17 @@ impl Host for Ssh {
} }
async fn get_main_system_profile(&mut self) -> ColmenaResult<Profile> { async fn get_main_system_profile(&mut self) -> ColmenaResult<Profile> {
let command = format!("\"readlink -e {} || readlink -e {}\"", SYSTEM_PROFILE, CURRENT_PROFILE); let command = format!(
"\"readlink -e {} || readlink -e {}\"",
SYSTEM_PROFILE, CURRENT_PROFILE
);
let paths = self.ssh(&["sh", "-c", &command]) let paths = self.ssh(&["sh", "-c", &command]).capture_output().await?;
.capture_output()
.await?;
let path = paths.lines().into_iter().next() let path = paths
.lines()
.into_iter()
.next()
.ok_or(ColmenaError::FailedToGetCurrentProfile)? .ok_or(ColmenaError::FailedToGetCurrentProfile)?
.to_string() .to_string()
.try_into()?; .try_into()?;
@ -151,9 +171,7 @@ impl Host for Ssh {
let profile = self.get_current_system_profile().await?; let profile = self.get_current_system_profile().await?;
if new_profile != profile { if new_profile != profile {
return Err(ColmenaError::ActiveProfileUnexpected { return Err(ColmenaError::ActiveProfileUnexpected { profile });
profile,
});
} }
} }
@ -201,8 +219,7 @@ impl Ssh {
let mut cmd = Command::new("ssh"); let mut cmd = Command::new("ssh");
cmd cmd.arg(self.ssh_target())
.arg(self.ssh_target())
.args(&options) .args(&options)
.arg("--") .arg("--")
.args(privilege_escalation_command) .args(privilege_escalation_command)
@ -226,7 +243,12 @@ impl Ssh {
} }
} }
fn nix_copy_closure(&self, path: &StorePath, direction: CopyDirection, options: CopyOptions) -> Command { fn nix_copy_closure(
&self,
path: &StorePath,
direction: CopyDirection,
options: CopyOptions,
) -> Command {
let ssh_options = self.ssh_options(); let ssh_options = self.ssh_options();
let ssh_options_str = ssh_options.join(" "); let ssh_options_str = ssh_options.join(" ");
@ -262,8 +284,16 @@ impl Ssh {
fn ssh_options(&self) -> Vec<String> { fn ssh_options(&self) -> Vec<String> {
// TODO: Allow configuation of SSH parameters // TODO: Allow configuation of SSH parameters
let mut options: Vec<String> = ["-o", "StrictHostKeyChecking=accept-new", "-o", "BatchMode=yes", "-T"] let mut options: Vec<String> = [
.iter().map(|s| s.to_string()).collect(); "-o",
"StrictHostKeyChecking=accept-new",
"-o",
"BatchMode=yes",
"-T",
]
.iter()
.map(|s| s.to_string())
.collect();
if let Some(port) = self.port { if let Some(port) = self.port {
options.push("-p".to_string()); options.push("-p".to_string());
@ -279,7 +309,12 @@ impl Ssh {
} }
/// Uploads a single key. /// Uploads a single key.
async fn upload_key(&mut self, name: &str, key: &Key, require_ownership: bool) -> ColmenaResult<()> { async fn upload_key(
&mut self,
name: &str,
key: &Key,
require_ownership: bool,
) -> ColmenaResult<()> {
if let Some(job) = &self.job { if let Some(job) = &self.job {
job.message(format!("Uploading key {}", name))?; job.message(format!("Uploading key {}", name))?;
} }
@ -299,7 +334,8 @@ impl Ssh {
/// Returns the current Boot ID. /// Returns the current Boot ID.
async fn get_boot_id(&mut self) -> ColmenaResult<BootId> { async fn get_boot_id(&mut self) -> ColmenaResult<BootId> {
let boot_id = self.ssh(&["cat", "/proc/sys/kernel/random/boot_id"]) let boot_id = self
.ssh(&["cat", "/proc/sys/kernel/random/boot_id"])
.capture_output() .capture_output()
.await?; .await?;

View file

@ -20,7 +20,11 @@ impl NixVersion {
let major = caps.name("major").unwrap().as_str().parse().unwrap(); let major = caps.name("major").unwrap().as_str().parse().unwrap();
let minor = caps.name("minor").unwrap().as_str().parse().unwrap(); let minor = caps.name("minor").unwrap().as_str().parse().unwrap();
Self { major, minor, string } Self {
major,
minor,
string,
}
} else { } else {
Self { Self {
major: 0, major: 0,
@ -61,20 +65,23 @@ impl NixCheck {
pub async fn detect() -> Self { pub async fn detect() -> Self {
let version_cmd = Command::new("nix-instantiate") let version_cmd = Command::new("nix-instantiate")
.arg("--version") .arg("--version")
.output().await; .output()
.await;
if version_cmd.is_err() { if version_cmd.is_err() {
return Self::NO_NIX; return Self::NO_NIX;
} }
let version = NixVersion::parse(String::from_utf8_lossy(&version_cmd.unwrap().stdout).to_string()); let version =
NixVersion::parse(String::from_utf8_lossy(&version_cmd.unwrap().stdout).to_string());
let flakes_supported = version.has_flakes(); let flakes_supported = version.has_flakes();
let flake_cmd = Command::new("nix-instantiate") let flake_cmd = Command::new("nix-instantiate")
.args(&["--eval", "-E", "builtins.getFlake"]) .args(&["--eval", "-E", "builtins.getFlake"])
.stdout(Stdio::null()) .stdout(Stdio::null())
.stderr(Stdio::null()) .stderr(Stdio::null())
.status().await; .status()
.await;
if flake_cmd.is_err() { if flake_cmd.is_err() {
return Self::NO_NIX; return Self::NO_NIX;
@ -121,16 +128,18 @@ impl NixCheck {
log::warn!("Colmena will automatically enable Flakes for its operations, but you should enable it in your Nix configuration:"); log::warn!("Colmena will automatically enable Flakes for its operations, but you should enable it in your Nix configuration:");
log::warn!(" experimental-features = nix-command flakes"); log::warn!(" experimental-features = nix-command flakes");
} else { } else {
let level = if required { let level = if required { Level::Error } else { Level::Warn };
Level::Error log::log!(
} else { level,
Level::Warn "The Nix version you are using does not support Flakes."
}; );
log::log!(level, "The Nix version you are using does not support Flakes.");
log::log!(level, "If you are using a Nixpkgs version before 21.11, please install nixUnstable for a version that includes Flakes support."); log::log!(level, "If you are using a Nixpkgs version before 21.11, please install nixUnstable for a version that includes Flakes support.");
if required { if required {
log::log!(level, "Cannot continue since Flakes support is required for this operation."); log::log!(
level,
"Cannot continue since Flakes support is required for this operation."
);
} }
} }
} }

View file

@ -8,11 +8,7 @@ use std::{
use regex::Regex; use regex::Regex;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use snafu::Snafu; use snafu::Snafu;
use tokio::{ use tokio::{fs::File, io::AsyncRead, process::Command};
fs::File,
io::AsyncRead,
process::Command,
};
use validator::{Validate, ValidationError}; use validator::{Validate, ValidationError};
#[non_exhaustive] #[non_exhaustive]
@ -48,18 +44,13 @@ impl TryFrom<KeySources> for KeySource {
fn try_from(ks: KeySources) -> Result<Self, Self::Error> { fn try_from(ks: KeySources) -> Result<Self, Self::Error> {
match (ks.text, ks.command, ks.file) { match (ks.text, ks.command, ks.file) {
(Some(text), None, None) => { (Some(text), None, None) => Ok(KeySource::Text(text)),
Ok(KeySource::Text(text)) (None, Some(command), None) => Ok(KeySource::Command(command)),
} (None, None, Some(file)) => Ok(KeySource::File(file)),
(None, Some(command), None) => { x => Err(format!(
Ok(KeySource::Command(command)) "Somehow 0 or more than 1 key source was specified: {:?}",
} x
(None, None, Some(file)) => { )),
Ok(KeySource::File(file))
}
x => {
Err(format!("Somehow 0 or more than 1 key source was specified: {:?}", x))
}
} }
} }
} }
@ -115,9 +106,7 @@ pub struct Key {
impl Key { impl Key {
pub async fn reader(&'_ self) -> Result<Box<dyn AsyncRead + Send + Unpin + '_>, KeyError> { pub async fn reader(&'_ self) -> Result<Box<dyn AsyncRead + Send + Unpin + '_>, KeyError> {
match &self.source { match &self.source {
KeySource::Text(content) => { KeySource::Text(content) => Ok(Box::new(Cursor::new(content))),
Ok(Box::new(Cursor::new(content)))
}
KeySource::Command(command) => { KeySource::Command(command) => {
let pathname = &command[0]; let pathname = &command[0];
let argv = &command[1..]; let argv = &command[1..];
@ -128,7 +117,8 @@ impl Key {
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()) .stderr(Stdio::piped())
.spawn()? .spawn()?
.wait_with_output().await?; .wait_with_output()
.await?;
if output.status.success() { if output.status.success() {
Ok(Box::new(Cursor::new(output.stdout))) Ok(Box::new(Cursor::new(output.stdout)))
@ -142,18 +132,28 @@ impl Key {
}) })
} }
} }
KeySource::File(path) => { KeySource::File(path) => Ok(Box::new(File::open(path).await?)),
Ok(Box::new(File::open(path).await?))
}
} }
} }
pub fn name(&self) -> &str { &self.name } pub fn name(&self) -> &str {
pub fn path(&self) -> &Path { &self.path } &self.name
pub fn user(&self) -> &str { &self.user } }
pub fn group(&self) -> &str { &self.group } pub fn path(&self) -> &Path {
pub fn permissions(&self) -> &str { &self.permissions } &self.path
pub fn upload_at(&self) -> UploadAt { self.upload_at } }
pub fn user(&self) -> &str {
&self.user
}
pub fn group(&self) -> &str {
&self.group
}
pub fn permissions(&self) -> &str {
&self.permissions
}
pub fn upload_at(&self) -> UploadAt {
self.upload_at
}
} }
fn validate_unix_name(name: &str) -> Result<(), ValidationError> { fn validate_unix_name(name: &str) -> Result<(), ValidationError> {
@ -169,6 +169,8 @@ fn validate_dest_dir(dir: &Path) -> Result<(), ValidationError> {
if dir.has_root() { if dir.has_root() {
Ok(()) Ok(())
} else { } else {
Err(ValidationError::new("Secret key destination directory must be absolute")) Err(ValidationError::new(
"Secret key destination directory must be absolute",
))
} }
} }

View file

@ -7,17 +7,17 @@ use serde::de;
use serde::{Deserialize, Deserializer, Serialize}; use serde::{Deserialize, Deserializer, Serialize};
use validator::{Validate, ValidationError as ValidationErrorType}; use validator::{Validate, ValidationError as ValidationErrorType};
use crate::error::{ColmenaResult, ColmenaError}; use crate::error::{ColmenaError, ColmenaResult};
pub mod host; pub mod host;
pub use host::{Host, CopyDirection, CopyOptions, RebootOptions};
use host::Ssh; use host::Ssh;
pub use host::{CopyDirection, CopyOptions, Host, RebootOptions};
pub mod hive; pub mod hive;
pub use hive::{Hive, HivePath}; pub use hive::{Hive, HivePath};
pub mod store; pub mod store;
pub use store::{StorePath, StoreDerivation, BuildResult}; pub use store::{BuildResult, StoreDerivation, StorePath};
pub mod key; pub mod key;
pub use key::Key; pub use key::Key;
@ -48,10 +48,7 @@ pub const CURRENT_PROFILE: &str = "/run/current-system";
/// A node's attribute name. /// A node's attribute name.
#[derive(Serialize, Deserialize, Clone, Debug, Hash, Eq, PartialEq)] #[derive(Serialize, Deserialize, Clone, Debug, Hash, Eq, PartialEq)]
#[serde(transparent)] #[serde(transparent)]
pub struct NodeName ( pub struct NodeName(#[serde(deserialize_with = "NodeName::deserialize")] String);
#[serde(deserialize_with = "NodeName::deserialize")]
String
);
#[derive(Debug, Clone, Validate, Deserialize)] #[derive(Debug, Clone, Validate, Deserialize)]
pub struct NodeConfig { pub struct NodeConfig {
@ -108,7 +105,7 @@ pub struct NixOptions {
} }
/// A Nix expression. /// A Nix expression.
pub trait NixExpression : Send + Sync { pub trait NixExpression: Send + Sync {
/// Returns the full Nix expression to be evaluated. /// Returns the full Nix expression to be evaluated.
fn expression(&self) -> String; fn expression(&self) -> String;
@ -132,13 +129,12 @@ impl NodeName {
/// Deserializes a potentially-invalid node name. /// Deserializes a potentially-invalid node name.
fn deserialize<'de, D>(deserializer: D) -> Result<String, D::Error> fn deserialize<'de, D>(deserializer: D) -> Result<String, D::Error>
where D: Deserializer<'de> where
D: Deserializer<'de>,
{ {
use de::Error; use de::Error;
String::deserialize(deserializer) String::deserialize(deserializer)
.and_then(|s| { .and_then(|s| Self::validate(s).map_err(|e| Error::custom(e.to_string())))
Self::validate(s).map_err(|e| Error::custom(e.to_string()))
})
} }
fn validate(s: String) -> ColmenaResult<String> { fn validate(s: String) -> ColmenaResult<String> {
@ -160,14 +156,22 @@ impl Deref for NodeName {
} }
impl NodeConfig { impl NodeConfig {
pub fn tags(&self) -> &[String] { &self.tags } pub fn tags(&self) -> &[String] {
&self.tags
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))] #[cfg_attr(not(target_os = "linux"), allow(dead_code))]
pub fn allows_local_deployment(&self) -> bool { self.allow_local_deployment } pub fn allows_local_deployment(&self) -> bool {
self.allow_local_deployment
}
pub fn privilege_escalation_command(&self) -> &Vec<String> { &self.privilege_escalation_command } pub fn privilege_escalation_command(&self) -> &Vec<String> {
&self.privilege_escalation_command
}
pub fn build_on_target(&self) -> bool { self.build_on_target } pub fn build_on_target(&self) -> bool {
self.build_on_target
}
pub fn set_build_on_target(&mut self, enable: bool) { pub fn set_build_on_target(&mut self, enable: bool) {
self.build_on_target = enable; self.build_on_target = enable;
} }
@ -228,11 +232,15 @@ fn validate_keys(keys: &HashMap<String, Key>) -> Result<(), ValidationErrorType>
for name in keys.keys() { for name in keys.keys() {
let path = Path::new(name); let path = Path::new(name);
if path.has_root() { if path.has_root() {
return Err(ValidationErrorType::new("Secret key name cannot be absolute")); return Err(ValidationErrorType::new(
"Secret key name cannot be absolute",
));
} }
if path.components().count() != 1 { if path.components().count() != 1 {
return Err(ValidationErrorType::new("Secret key name cannot contain path separators")); return Err(ValidationErrorType::new(
"Secret key name cannot contain path separators",
));
} }
} }
Ok(()) Ok(())

View file

@ -2,11 +2,11 @@
use std::collections::HashSet; use std::collections::HashSet;
use std::convert::AsRef; use std::convert::AsRef;
use std::iter::{Iterator, FromIterator}; use std::iter::{FromIterator, Iterator};
use glob::Pattern as GlobPattern; use glob::Pattern as GlobPattern;
use super::{ColmenaError, ColmenaResult, NodeName, NodeConfig}; use super::{ColmenaError, ColmenaResult, NodeConfig, NodeName};
/// A node filter containing a list of rules. /// A node filter containing a list of rules.
pub struct NodeFilter { pub struct NodeFilter {
@ -34,30 +34,29 @@ impl NodeFilter {
if trimmed.is_empty() { if trimmed.is_empty() {
log::warn!("Filter \"{}\" is blank and will match nothing", filter); log::warn!("Filter \"{}\" is blank and will match nothing", filter);
return Ok(Self { return Ok(Self { rules: Vec::new() });
rules: Vec::new(),
});
} }
let rules = trimmed.split(',').map(|pattern| { let rules = trimmed
let pattern = pattern.trim(); .split(',')
.map(|pattern| {
let pattern = pattern.trim();
if pattern.is_empty() { if pattern.is_empty() {
return Err(ColmenaError::EmptyFilterRule); return Err(ColmenaError::EmptyFilterRule);
} }
if let Some(tag_pattern) = pattern.strip_prefix('@') { if let Some(tag_pattern) = pattern.strip_prefix('@') {
Ok(Rule::MatchTag(GlobPattern::new(tag_pattern).unwrap())) Ok(Rule::MatchTag(GlobPattern::new(tag_pattern).unwrap()))
} else { } else {
Ok(Rule::MatchName(GlobPattern::new(pattern).unwrap())) Ok(Rule::MatchName(GlobPattern::new(pattern).unwrap()))
} }
}).collect::<Vec<ColmenaResult<Rule>>>(); })
.collect::<Vec<ColmenaResult<Rule>>>();
let rules = Result::from_iter(rules)?; let rules = Result::from_iter(rules)?;
Ok(Self { Ok(Self { rules })
rules,
})
} }
/// Returns whether the filter has any rule matching NodeConfig information. /// Returns whether the filter has any rule matching NodeConfig information.
@ -71,32 +70,36 @@ impl NodeFilter {
/// Runs the filter against a set of NodeConfigs and returns the matched ones. /// Runs the filter against a set of NodeConfigs and returns the matched ones.
pub fn filter_node_configs<'a, I>(&self, nodes: I) -> HashSet<NodeName> pub fn filter_node_configs<'a, I>(&self, nodes: I) -> HashSet<NodeName>
where I: Iterator<Item = (&'a NodeName, &'a NodeConfig)> where
I: Iterator<Item = (&'a NodeName, &'a NodeConfig)>,
{ {
if self.rules.is_empty() { if self.rules.is_empty() {
return HashSet::new(); return HashSet::new();
} }
nodes.filter_map(|(name, node)| { nodes
for rule in self.rules.iter() { .filter_map(|(name, node)| {
match rule { for rule in self.rules.iter() {
Rule::MatchName(pat) => { match rule {
if pat.matches(name.as_str()) { Rule::MatchName(pat) => {
return Some(name); if pat.matches(name.as_str()) {
}
}
Rule::MatchTag(pat) => {
for tag in node.tags() {
if pat.matches(tag) {
return Some(name); return Some(name);
} }
} }
Rule::MatchTag(pat) => {
for tag in node.tags() {
if pat.matches(tag) {
return Some(name);
}
}
}
} }
} }
}
None None
}).cloned().collect() })
.cloned()
.collect()
} }
/// Runs the filter against a set of node names and returns the matched ones. /// Runs the filter against a set of node names and returns the matched ones.
@ -140,7 +143,7 @@ mod tests {
macro_rules! node { macro_rules! node {
($n:expr) => { ($n:expr) => {
NodeName::new($n.to_string()).unwrap() NodeName::new($n.to_string()).unwrap()
} };
} }
#[test] #[test]
@ -186,16 +189,22 @@ mod tests {
#[test] #[test]
fn test_filter_node_names() { fn test_filter_node_names() {
let nodes = vec![ node!("lax-alpha"), node!("lax-beta"), node!("sfo-gamma") ]; let nodes = vec![node!("lax-alpha"), node!("lax-beta"), node!("sfo-gamma")];
assert_eq!( assert_eq!(
&HashSet::from_iter([ node!("lax-alpha") ]), &HashSet::from_iter([node!("lax-alpha")]),
&NodeFilter::new("lax-alpha").unwrap().filter_node_names(&nodes).unwrap(), &NodeFilter::new("lax-alpha")
.unwrap()
.filter_node_names(&nodes)
.unwrap(),
); );
assert_eq!( assert_eq!(
&HashSet::from_iter([ node!("lax-alpha"), node!("lax-beta") ]), &HashSet::from_iter([node!("lax-alpha"), node!("lax-beta")]),
&NodeFilter::new("lax-*").unwrap().filter_node_names(&nodes).unwrap(), &NodeFilter::new("lax-*")
.unwrap()
.filter_node_names(&nodes)
.unwrap(),
); );
} }
@ -216,46 +225,66 @@ mod tests {
let mut nodes = HashMap::new(); let mut nodes = HashMap::new();
nodes.insert(node!("alpha"), NodeConfig { nodes.insert(
tags: vec![ "web".to_string(), "infra-lax".to_string() ], node!("alpha"),
..template.clone() NodeConfig {
}); tags: vec!["web".to_string(), "infra-lax".to_string()],
..template.clone()
},
);
nodes.insert(node!("beta"), NodeConfig { nodes.insert(
tags: vec![ "router".to_string(), "infra-sfo".to_string() ], node!("beta"),
..template.clone() NodeConfig {
}); tags: vec!["router".to_string(), "infra-sfo".to_string()],
..template.clone()
},
);
nodes.insert(node!("gamma-a"), NodeConfig { nodes.insert(
tags: vec![ "controller".to_string() ], node!("gamma-a"),
..template.clone() NodeConfig {
}); tags: vec!["controller".to_string()],
..template.clone()
},
);
nodes.insert(node!("gamma-b"), NodeConfig { nodes.insert(
tags: vec![ "ewaste".to_string() ], node!("gamma-b"),
..template NodeConfig {
}); tags: vec!["ewaste".to_string()],
..template
},
);
assert_eq!(4, nodes.len()); assert_eq!(4, nodes.len());
assert_eq!( assert_eq!(
&HashSet::from_iter([ node!("alpha") ]), &HashSet::from_iter([node!("alpha")]),
&NodeFilter::new("@web").unwrap().filter_node_configs(nodes.iter()), &NodeFilter::new("@web")
.unwrap()
.filter_node_configs(nodes.iter()),
); );
assert_eq!( assert_eq!(
&HashSet::from_iter([ node!("alpha"), node!("beta") ]), &HashSet::from_iter([node!("alpha"), node!("beta")]),
&NodeFilter::new("@infra-*").unwrap().filter_node_configs(nodes.iter()), &NodeFilter::new("@infra-*")
.unwrap()
.filter_node_configs(nodes.iter()),
); );
assert_eq!( assert_eq!(
&HashSet::from_iter([ node!("beta"), node!("gamma-a") ]), &HashSet::from_iter([node!("beta"), node!("gamma-a")]),
&NodeFilter::new("@router,@controller").unwrap().filter_node_configs(nodes.iter()), &NodeFilter::new("@router,@controller")
.unwrap()
.filter_node_configs(nodes.iter()),
); );
assert_eq!( assert_eq!(
&HashSet::from_iter([ node!("beta"), node!("gamma-a"), node!("gamma-b") ]), &HashSet::from_iter([node!("beta"), node!("gamma-a"), node!("gamma-b")]),
&NodeFilter::new("@router,gamma-*").unwrap().filter_node_configs(nodes.iter()), &NodeFilter::new("@router,gamma-*")
.unwrap()
.filter_node_configs(nodes.iter()),
); );
} }
} }

View file

@ -4,14 +4,7 @@ use std::process::Stdio;
use tokio::process::Command; use tokio::process::Command;
use super::{ use super::{BuildResult, ColmenaError, ColmenaResult, Goal, StoreDerivation, StorePath};
Goal,
ColmenaResult,
ColmenaError,
StorePath,
StoreDerivation,
BuildResult,
};
pub type ProfileDerivation = StoreDerivation<Profile>; pub type ProfileDerivation = StoreDerivation<Profile>;
@ -21,10 +14,7 @@ pub struct Profile(StorePath);
impl Profile { impl Profile {
pub fn from_store_path(path: StorePath) -> ColmenaResult<Self> { pub fn from_store_path(path: StorePath) -> ColmenaResult<Self> {
if if !path.is_dir() || !path.join("bin/switch-to-configuration").exists() {
!path.is_dir() ||
!path.join("bin/switch-to-configuration").exists()
{
return Err(ColmenaError::InvalidProfile); return Err(ColmenaError::InvalidProfile);
} }
@ -39,14 +29,12 @@ impl Profile {
pub fn activation_command(&self, goal: Goal) -> Option<Vec<String>> { pub fn activation_command(&self, goal: Goal) -> Option<Vec<String>> {
if let Some(goal) = goal.as_str() { if let Some(goal) = goal.as_str() {
let path = self.as_path().join("bin/switch-to-configuration"); let path = self.as_path().join("bin/switch-to-configuration");
let switch_to_configuration = path.to_str() let switch_to_configuration = path
.to_str()
.expect("The string should be UTF-8 valid") .expect("The string should be UTF-8 valid")
.to_string(); .to_string();
Some(vec![ Some(vec![switch_to_configuration, goal.to_string()])
switch_to_configuration,
goal.to_string(),
])
} else { } else {
None None
} }
@ -65,7 +53,12 @@ impl Profile {
/// Create a GC root for this profile. /// Create a GC root for this profile.
pub async fn create_gc_root(&self, path: &Path) -> ColmenaResult<()> { pub async fn create_gc_root(&self, path: &Path) -> ColmenaResult<()> {
let mut command = Command::new("nix-store"); let mut command = Command::new("nix-store");
command.args(&["--no-build-output", "--indirect", "--add-root", path.to_str().unwrap()]); command.args(&[
"--no-build-output",
"--indirect",
"--add-root",
path.to_str().unwrap(),
]);
command.args(&["--realise", self.as_path().to_str().unwrap()]); command.args(&["--realise", self.as_path().to_str().unwrap()]);
command.stdout(Stdio::null()); command.stdout(Stdio::null());
@ -100,8 +93,7 @@ impl TryFrom<BuildResult<Profile>> for Profile {
}); });
} }
let path = paths.iter().next() let path = paths.iter().next().unwrap().to_owned();
.unwrap().to_owned();
Ok(Self::from_store_path_unchecked(path)) Ok(Self::from_store_path_unchecked(path))
} }

View file

@ -1,15 +1,15 @@
use std::convert::{TryFrom, TryInto}; use std::convert::{TryFrom, TryInto};
use std::marker::PhantomData;
use std::path::{Path, PathBuf};
use std::ops::Deref;
use std::fmt; use std::fmt;
use std::marker::PhantomData;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use serde::{Serialize, Deserialize}; use serde::{Deserialize, Serialize};
use tokio::process::Command; use tokio::process::Command;
use super::Host;
use crate::error::{ColmenaError, ColmenaResult}; use crate::error::{ColmenaError, ColmenaResult};
use crate::util::CommandExt; use crate::util::CommandExt;
use super::Host;
/// A Nix store path. /// A Nix store path.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@ -17,7 +17,7 @@ pub struct StorePath(PathBuf);
/// A store derivation (.drv) that will result in a T when built. /// A store derivation (.drv) that will result in a T when built.
#[derive(Debug)] #[derive(Debug)]
pub struct StoreDerivation<T: TryFrom<BuildResult<T>>>{ pub struct StoreDerivation<T: TryFrom<BuildResult<T>>> {
path: StorePath, path: StorePath,
_target: PhantomData<T>, _target: PhantomData<T>,
} }
@ -48,9 +48,12 @@ impl StorePath {
let references = Command::new("nix-store") let references = Command::new("nix-store")
.args(&["--query", "--references"]) .args(&["--query", "--references"])
.arg(&self.0) .arg(&self.0)
.capture_output().await? .capture_output()
.trim_end().split('\n') .await?
.map(|p| StorePath(PathBuf::from(p))).collect(); .trim_end()
.split('\n')
.map(|p| StorePath(PathBuf::from(p)))
.collect();
Ok(references) Ok(references)
} }
@ -114,7 +117,7 @@ impl<T: TryFrom<BuildResult<T>>> StoreDerivation<T> {
} }
} }
impl<T: TryFrom<BuildResult<T>, Error=ColmenaError>> StoreDerivation<T> { impl<T: TryFrom<BuildResult<T>, Error = ColmenaError>> StoreDerivation<T> {
/// Builds the store derivation on a host, resulting in a T. /// Builds the store derivation on a host, resulting in a T.
pub async fn realize(&self, host: &mut Box<dyn Host>) -> ColmenaResult<T> { pub async fn realize(&self, host: &mut Box<dyn Host>) -> ColmenaResult<T> {
let paths: Vec<StorePath> = host.realize(&self.path).await?; let paths: Vec<StorePath> = host.realize(&self.path).await?;
@ -144,7 +147,7 @@ impl<T: TryFrom<BuildResult<T>>> fmt::Display for StoreDerivation<T> {
} }
} }
impl<T: TryFrom<BuildResult<T>, Error=ColmenaError>> BuildResult<T> { impl<T: TryFrom<BuildResult<T>, Error = ColmenaError>> BuildResult<T> {
pub fn paths(&self) -> &[StorePath] { pub fn paths(&self) -> &[StorePath] {
self.results.as_slice() self.results.as_slice()
} }

View file

@ -8,10 +8,7 @@ pub mod plain;
pub mod spinner; pub mod spinner;
use async_trait::async_trait; use async_trait::async_trait;
use tokio::sync::mpsc::{self, use tokio::sync::mpsc::{self, UnboundedReceiver as TokioReceiver, UnboundedSender as TokioSender};
UnboundedReceiver as TokioReceiver,
UnboundedSender as TokioSender,
};
use crate::error::ColmenaResult; use crate::error::ColmenaResult;
use crate::job::JobId; use crate::job::JobId;
@ -31,7 +28,7 @@ pub enum SimpleProgressOutput {
/// A progress display driver. /// A progress display driver.
#[async_trait] #[async_trait]
pub trait ProgressOutput : Sized { pub trait ProgressOutput: Sized {
/// Runs until a Message::Complete is received. /// Runs until a Message::Complete is received.
async fn run_until_completion(self) -> ColmenaResult<Self>; async fn run_until_completion(self) -> ColmenaResult<Self>;
@ -111,14 +108,8 @@ impl SimpleProgressOutput {
pub async fn run_until_completion(self) -> ColmenaResult<Self> { pub async fn run_until_completion(self) -> ColmenaResult<Self> {
match self { match self {
Self::Plain(o) => { Self::Plain(o) => o.run_until_completion().await.map(Self::Plain),
o.run_until_completion().await Self::Spinner(o) => o.run_until_completion().await.map(Self::Spinner),
.map(Self::Plain)
}
Self::Spinner(o) => {
o.run_until_completion().await
.map(Self::Spinner)
}
} }
} }
} }

View file

@ -3,17 +3,10 @@
use async_trait::async_trait; use async_trait::async_trait;
use console::Style as ConsoleStyle; use console::Style as ConsoleStyle;
use crate::error::ColmenaResult;
use super::{ use super::{
DEFAULT_LABEL_WIDTH, create_channel, Line, LineStyle, Message, ProgressOutput, Receiver, Sender, DEFAULT_LABEL_WIDTH,
ProgressOutput,
Sender,
Receiver,
Message,
Line,
LineStyle,
create_channel,
}; };
use crate::error::ColmenaResult;
pub struct PlainOutput { pub struct PlainOutput {
sender: Option<Sender>, sender: Option<Sender>,
@ -42,36 +35,21 @@ impl PlainOutput {
} }
let label_style = match line.style { let label_style = match line.style {
LineStyle::Normal => { LineStyle::Normal => ConsoleStyle::new().bold(),
ConsoleStyle::new().bold() LineStyle::Success => ConsoleStyle::new().bold().green(),
} LineStyle::SuccessNoop => ConsoleStyle::new().bold().green().dim(),
LineStyle::Success => { LineStyle::Failure => ConsoleStyle::new().bold().red(),
ConsoleStyle::new().bold().green()
}
LineStyle::SuccessNoop => {
ConsoleStyle::new().bold().green().dim()
}
LineStyle::Failure => {
ConsoleStyle::new().bold().red()
}
}; };
let text_style = match line.style { let text_style = match line.style {
LineStyle::Normal => { LineStyle::Normal => ConsoleStyle::new(),
ConsoleStyle::new() LineStyle::Success => ConsoleStyle::new().green(),
} LineStyle::SuccessNoop => ConsoleStyle::new().dim(),
LineStyle::Success => { LineStyle::Failure => ConsoleStyle::new().red(),
ConsoleStyle::new().green()
}
LineStyle::SuccessNoop => {
ConsoleStyle::new().dim()
}
LineStyle::Failure => {
ConsoleStyle::new().red()
}
}; };
eprintln!("{:>width$} | {}", eprintln!(
"{:>width$} | {}",
label_style.apply_to(line.label), label_style.apply_to(line.label),
text_style.apply_to(line.text), text_style.apply_to(line.text),
width = self.label_width, width = self.label_width,

View file

@ -4,20 +4,13 @@ use std::collections::HashMap;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use async_trait::async_trait; use async_trait::async_trait;
use indicatif::{MultiProgress, ProgressStyle, ProgressBar}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
use super::{
create_channel, Line, LineStyle, Message, ProgressOutput, Receiver, Sender, DEFAULT_LABEL_WIDTH,
};
use crate::error::ColmenaResult; use crate::error::ColmenaResult;
use crate::job::JobId; use crate::job::JobId;
use super::{
DEFAULT_LABEL_WIDTH,
ProgressOutput,
Sender,
Receiver,
Message,
Line,
LineStyle,
create_channel,
};
/// Progress spinner output. /// Progress spinner output.
pub struct SpinnerOutput { pub struct SpinnerOutput {
@ -91,8 +84,7 @@ impl SpinnerOutput {
/// Creates a new bar. /// Creates a new bar.
fn create_bar(&self, style: LineStyle) -> ProgressBar { fn create_bar(&self, style: LineStyle) -> ProgressBar {
let bar = ProgressBar::new(100) let bar = ProgressBar::new(100).with_style(self.get_spinner_style(style));
.with_style(self.get_spinner_style(style));
let bar = self.multi.add(bar); let bar = self.multi.add(bar);
bar.enable_steady_tick(Duration::from_millis(100)); bar.enable_steady_tick(Duration::from_millis(100));
@ -222,25 +214,27 @@ impl JobState {
} }
fn configure_one_off(&self, bar: &ProgressBar) { fn configure_one_off(&self, bar: &ProgressBar) {
bar.clone().with_elapsed(Instant::now().duration_since(self.since)); bar.clone()
.with_elapsed(Instant::now().duration_since(self.since));
} }
} }
fn get_spinner_style(label_width: usize, style: LineStyle) -> ProgressStyle { fn get_spinner_style(label_width: usize, style: LineStyle) -> ProgressStyle {
let template = format!("{{prefix:>{}.bold.dim}} {{spinner}} {{elapsed}} {{wide_msg}}", label_width); let template = format!(
"{{prefix:>{}.bold.dim}} {{spinner}} {{elapsed}} {{wide_msg}}",
label_width
);
match style { match style {
LineStyle::Normal | LineStyle::Success | LineStyle::SuccessNoop => { LineStyle::Normal | LineStyle::Success | LineStyle::SuccessNoop => {
ProgressStyle::default_spinner() ProgressStyle::default_spinner()
.tick_chars("🕛🕐🕑🕒🕓🕔🕕🕖🕗🕘🕙🕚✅") .tick_chars("🕛🕐🕑🕒🕓🕔🕕🕖🕗🕘🕙🕚✅")
.template(&template) .template(&template)
.unwrap() .unwrap()
} }
LineStyle::Failure => { LineStyle::Failure => ProgressStyle::default_spinner()
ProgressStyle::default_spinner()
.tick_chars("❌❌") .tick_chars("❌❌")
.template(&template) .template(&template)
.unwrap() .unwrap(),
}
} }
} }

View file

@ -10,9 +10,14 @@ use clap::ArgMatches;
use crate::error::ColmenaError; use crate::error::ColmenaError;
/// Runs a closure and tries to troubleshoot if it returns an error. /// Runs a closure and tries to troubleshoot if it returns an error.
pub async fn run_wrapped<'a, F, U, T>(global_args: &'a ArgMatches, local_args: &'a ArgMatches, f: U) -> T pub async fn run_wrapped<'a, F, U, T>(
where U: FnOnce(&'a ArgMatches, &'a ArgMatches) -> F, global_args: &'a ArgMatches,
F: Future<Output = Result<T, ColmenaError>>, local_args: &'a ArgMatches,
f: U,
) -> T
where
U: FnOnce(&'a ArgMatches, &'a ArgMatches) -> F,
F: Future<Output = Result<T, ColmenaError>>,
{ {
match f(global_args, local_args).await { match f(global_args, local_args).await {
Ok(r) => r, Ok(r) => r,
@ -21,16 +26,23 @@ pub async fn run_wrapped<'a, F, U, T>(global_args: &'a ArgMatches, local_args: &
log::error!("Operation failed with error: {}", error); log::error!("Operation failed with error: {}", error);
if let Err(own_error) = troubleshoot(global_args, local_args, &error) { if let Err(own_error) = troubleshoot(global_args, local_args, &error) {
log::error!("Error occurred while trying to troubleshoot another error: {}", own_error); log::error!(
"Error occurred while trying to troubleshoot another error: {}",
own_error
);
} }
// Ensure we exit with a code // Ensure we exit with a code
quit::with_code(1); quit::with_code(1);
}, }
} }
} }
fn troubleshoot(global_args: &ArgMatches, _local_args: &ArgMatches, error: &ColmenaError) -> Result<(), ColmenaError> { fn troubleshoot(
global_args: &ArgMatches,
_local_args: &ArgMatches,
error: &ColmenaError,
) -> Result<(), ColmenaError> {
if let ColmenaError::NoFlakesSupport = error { if let ColmenaError::NoFlakesSupport = error {
// People following the tutorial might put hive.nix directly // People following the tutorial might put hive.nix directly
// in their Colmena checkout, and encounter NoFlakesSupport // in their Colmena checkout, and encounter NoFlakesSupport
@ -39,7 +51,9 @@ fn troubleshoot(global_args: &ArgMatches, _local_args: &ArgMatches, error: &Colm
if global_args.occurrences_of("config") == 0 { if global_args.occurrences_of("config") == 0 {
let cwd = env::current_dir()?; let cwd = env::current_dir()?;
if cwd.join("flake.nix").is_file() && cwd.join("hive.nix").is_file() { if cwd.join("flake.nix").is_file() && cwd.join("hive.nix").is_file() {
eprintln!("Hint: You have both flake.nix and hive.nix in the current directory, and"); eprintln!(
"Hint: You have both flake.nix and hive.nix in the current directory, and"
);
eprintln!(" Colmena will always prefer flake.nix if it exists."); eprintln!(" Colmena will always prefer flake.nix if it exists.");
eprintln!(); eprintln!();
eprintln!(" Try passing `-f hive.nix` explicitly if this is what you want."); eprintln!(" Try passing `-f hive.nix` explicitly if this is what you want.");

View file

@ -3,16 +3,16 @@ use std::path::PathBuf;
use std::process::Stdio; use std::process::Stdio;
use async_trait::async_trait; use async_trait::async_trait;
use clap::{Command as ClapCommand, Arg, ArgMatches}; use clap::{Arg, ArgMatches, Command as ClapCommand};
use futures::future::join3; use futures::future::join3;
use serde::de::DeserializeOwned; use serde::de::DeserializeOwned;
use tokio::io::{AsyncRead, AsyncBufReadExt, BufReader}; use tokio::io::{AsyncBufReadExt, AsyncRead, BufReader};
use tokio::process::Command; use tokio::process::Command;
use super::error::{ColmenaResult, ColmenaError}; use super::error::{ColmenaError, ColmenaResult};
use super::nix::{Flake, Hive, HivePath, StorePath};
use super::nix::deployment::TargetNodeMap;
use super::job::JobHandle; use super::job::JobHandle;
use super::nix::deployment::TargetNodeMap;
use super::nix::{Flake, Hive, HivePath, StorePath};
const NEWLINE: u8 = 0xa; const NEWLINE: u8 = 0xa;
@ -35,7 +35,9 @@ pub trait CommandExt {
async fn capture_output(&mut self) -> ColmenaResult<String>; async fn capture_output(&mut self) -> ColmenaResult<String>;
/// Runs the command, capturing deserialized output from JSON. /// Runs the command, capturing deserialized output from JSON.
async fn capture_json<T>(&mut self) -> ColmenaResult<T> where T: DeserializeOwned; async fn capture_json<T>(&mut self) -> ColmenaResult<T>
where
T: DeserializeOwned;
/// Runs the command, capturing a single store path. /// Runs the command, capturing a single store path.
async fn capture_store_path(&mut self) -> ColmenaResult<StorePath>; async fn capture_store_path(&mut self) -> ColmenaResult<StorePath>;
@ -81,7 +83,11 @@ impl CommandExecution {
let stdout = BufReader::new(child.stdout.take().unwrap()); let stdout = BufReader::new(child.stdout.take().unwrap());
let stderr = BufReader::new(child.stderr.take().unwrap()); let stderr = BufReader::new(child.stderr.take().unwrap());
let stdout_job = if self.hide_stdout { None } else { self.job.clone() }; let stdout_job = if self.hide_stdout {
None
} else {
self.job.clone()
};
let futures = join3( let futures = join3(
capture_stream(stdout, stdout_job, false), capture_stream(stdout, stdout_job, false),
@ -107,10 +113,7 @@ impl CommandExecution {
impl CommandExt for Command { impl CommandExt for Command {
/// Runs the command with stdout and stderr passed through to the user. /// Runs the command with stdout and stderr passed through to the user.
async fn passthrough(&mut self) -> ColmenaResult<()> { async fn passthrough(&mut self) -> ColmenaResult<()> {
let exit = self let exit = self.spawn()?.wait().await?;
.spawn()?
.wait()
.await?;
if exit.success() { if exit.success() {
Ok(()) Ok(())
@ -138,10 +141,13 @@ impl CommandExt for Command {
} }
/// Captures deserialized output from JSON. /// Captures deserialized output from JSON.
async fn capture_json<T>(&mut self) -> ColmenaResult<T> where T: DeserializeOwned { async fn capture_json<T>(&mut self) -> ColmenaResult<T>
where
T: DeserializeOwned,
{
let output = self.capture_output().await?; let output = self.capture_output().await?;
serde_json::from_str(&output).map_err(|_| ColmenaError::BadOutput { serde_json::from_str(&output).map_err(|_| ColmenaError::BadOutput {
output: output.clone() output: output.clone(),
}) })
} }
@ -168,10 +174,13 @@ impl CommandExt for CommandExecution {
} }
/// Captures deserialized output from JSON. /// Captures deserialized output from JSON.
async fn capture_json<T>(&mut self) -> ColmenaResult<T> where T: DeserializeOwned { async fn capture_json<T>(&mut self) -> ColmenaResult<T>
where
T: DeserializeOwned,
{
let output = self.capture_output().await?; let output = self.capture_output().await?;
serde_json::from_str(&output).map_err(|_| ColmenaError::BadOutput { serde_json::from_str(&output).map_err(|_| ColmenaError::BadOutput {
output: output.clone() output: output.clone(),
}) })
} }
@ -214,13 +223,19 @@ pub async fn hive_from_args(args: &ArgMatches) -> ColmenaResult<Hive> {
} }
if file_path.is_none() { if file_path.is_none() {
log::error!("Could not find `hive.nix` or `flake.nix` in {:?} or any parent directory", std::env::current_dir()?); log::error!(
"Could not find `hive.nix` or `flake.nix` in {:?} or any parent directory",
std::env::current_dir()?
);
} }
file_path.unwrap() file_path.unwrap()
} }
_ => { _ => {
let path = args.value_of("config").expect("The config arg should exist").to_owned(); let path = args
.value_of("config")
.expect("The config arg should exist")
.to_owned();
let fpath = PathBuf::from(&path); let fpath = PathBuf::from(&path);
if !fpath.exists() && path.contains(':') { if !fpath.exists() && path.contains(':') {
@ -278,8 +293,13 @@ The list is comma-separated and globs are supported. To match tags, prepend the
.takes_value(true)) .takes_value(true))
} }
pub async fn capture_stream<R>(mut stream: BufReader<R>, job: Option<JobHandle>, stderr: bool) -> ColmenaResult<String> pub async fn capture_stream<R>(
where R: AsyncRead + Unpin mut stream: BufReader<R>,
job: Option<JobHandle>,
stderr: bool,
) -> ColmenaResult<String>
where
R: AsyncRead + Unpin,
{ {
let mut log = String::new(); let mut log = String::new();
@ -325,9 +345,7 @@ mod tests {
let expected = "Hello\nWorld\n"; let expected = "Hello\nWorld\n";
let stream = BufReader::new(expected.as_bytes()); let stream = BufReader::new(expected.as_bytes());
let captured = block_on(async { let captured = block_on(async { capture_stream(stream, None, false).await.unwrap() });
capture_stream(stream, None, false).await.unwrap()
});
assert_eq!(expected, captured); assert_eq!(expected, captured);
} }
@ -335,9 +353,7 @@ mod tests {
#[test] #[test]
fn test_capture_stream_with_invalid_utf8() { fn test_capture_stream_with_invalid_utf8() {
let stream = BufReader::new([0x80, 0xa].as_slice()); let stream = BufReader::new([0x80, 0xa].as_slice());
let captured = block_on(async { let captured = block_on(async { capture_stream(stream, None, false).await.unwrap() });
capture_stream(stream, None, false).await.unwrap()
});
assert_eq!("\u{fffd}\n", captured); assert_eq!("\u{fffd}\n", captured);
} }