ecoppens/colmena
1
0
Fork 0
forked from DGNum/colmena
Code Pull requests Activity

deployment: Fail the evaluation job if any attribute fail to evaluate

Browse source 
nix-eval-jobs doesn't return non-zero exit code if any attribute
fail to evaluate. Let's just keep track ourselves.

Fixes #122.
This commit is contained in:
Zhaofeng Li 2022-10-19 16:45:57 -06:00
parent 872949504b
commit 956c3363b8
2 changed files with 21 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/error.rs
View file

@ -34,6 +34,9 @@ pub enum ColmenaError {
#[snafu(display("Validation error"))]
ValidationError { errors: ValidationErrors },
#[snafu(display("Some attributes failed to evaluate"))]
AttributeEvaluationError,
#[snafu(display("Error processing key \"{}\": {}", name, error))]
KeyError { name: String, error: key::KeyError },

23
src/nix/deployment/mod.rs
View file

@ -239,7 +239,7 @@ impl Deployment {
let job = parent.create_job(JobType::Evaluate, nodes.clone())?;
let futures = job
let (futures, failed_attributes) = job
.run(|job| async move {
let mut evaluator = NixEvalJobs::default();
let eval_limit = self
@ -254,6 +254,7 @@ impl Deployment {
let mut stream = evaluator.evaluate(&expr, options).await?;
let mut futures: Vec<tokio::task::JoinHandle<ColmenaResult<()>>> = Vec::new();
let mut failed_attributes = Vec::new();
while let Some(item) = stream.next().await {
match item {
@ -308,13 +309,12 @@ impl Deployment {
EvalError::Attribute(e) => {
// Attribute-level error
//
// Here the eventual non-zero exit code of the evaluator
// will translate into an `EvalError::Global`, causing
// the entire future to resolve to an Err.
// We still let the rest of the evaluation finish but
// mark the whole Evaluate job as failed.
let node_name =
NodeName::new(e.attribute().to_string()).unwrap();
let nodes = vec![node_name];
let nodes = vec![node_name.clone()];
let job = parent.create_job(JobType::Evaluate, nodes)?;
job.state(JobState::Running)?;
@ -322,13 +322,20 @@ impl Deployment {
job.stderr(line.to_string())?;
}
job.state(JobState::Failed)?;
failed_attributes.push(node_name);
}
}
}
}
}
Ok(futures)
// HACK: Still return Ok() because we need to wait for existing jobs to finish
if !failed_attributes.is_empty() {
job.failure(&ColmenaError::AttributeEvaluationError)?;
}
Ok((futures, failed_attributes))
})
.await?;
@ -338,8 +345,12 @@ impl Deployment {
.map(|r| r.unwrap()) // panic on JoinError (future panicked)
.collect::<ColmenaResult<Vec<()>>>()?;
if !failed_attributes.is_empty() {
Err(ColmenaError::AttributeEvaluationError)
} else {
Ok(())
}
}
/// Executes the deployment against a portion of nodes.
async fn execute_one_chunk(