error: Add backtraces to ChildFailure and ChildKilled

The plan is to have more hierarchical error types so it's easier
to find exactly where in the deployment flow an error occurred.

Cargo.lock

@@ -2,6 +2,21 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "addr2line"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
+dependencies = [
+ "gimli",
+]
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
 [[package]]
 name = "aho-corasick"
 version = "0.7.19"
@@ -60,6 +75,21 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "backtrace"
+version = "0.3.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -347,6 +377,12 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "gimli"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
+
 [[package]]
 name = "glob"
 version = "0.3.0"
@@ -486,6 +522,15 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
+[[package]]
+name = "miniz_oxide"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
+dependencies = [
+ "adler",
+]
+
 [[package]]
 name = "mio"
 version = "0.8.4"
@@ -555,6 +600,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
 
+[[package]]
+name = "object"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.15.0"
@@ -688,6 +742,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
+
 [[package]]
 name = "ryu"
 version = "1.0.11"
@@ -755,6 +815,7 @@ version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5177903bf45656592d9eb5c0e22f408fc023aae51dbe2088889b71633ba451f2"
 dependencies = [
+ "backtrace",
  "doc-comment",
  "snafu-derive",
 ]

Cargo.toml

@@ -30,7 +30,7 @@ serde = { version = "1.0.118", features = ["derive"] }
 serde_json = "1.0"
 shell-escape = "0.1.5"
 sys-info = "0.9.0"
-snafu = "0.7.0"
+snafu = { version = "0.7.0", features = ["backtrace", "backtraces-impl-backtrace-crate"] }
 tempfile = "3.1.0"
 tokio-stream = "0.1.8"
 uuid = { version = "1.0.0", features = ["serde", "v4"] }

src/error.rs

@@ -3,7 +3,7 @@
 use std::os::unix::process::ExitStatusExt;
 use std::process::ExitStatus;
 
-use snafu::Snafu;
+use snafu::{Backtrace, Snafu};
 use validator::ValidationErrors;
 
 use crate::nix::{key, Profile, StorePath};
@@ -20,10 +20,13 @@ pub enum ColmenaError {
     BadOutput { output: String },
 
     #[snafu(display("Child process exited with error code: {}", exit_code))]
-    ChildFailure { exit_code: i32 },
+    ChildFailure {
+        exit_code: i32,
+        backtrace: Backtrace,
+    },
 
     #[snafu(display("Child process was killed by signal {}", signal))]
-    ChildKilled { signal: i32 },
+    ChildKilled { signal: i32, backtrace: Backtrace },
 
     #[snafu(display("This operation is not supported"))]
     Unsupported,
@@ -89,10 +92,11 @@ impl From<ValidationErrors> for ColmenaError {
 impl From<ExitStatus> for ColmenaError {
     fn from(status: ExitStatus) -> Self {
         match status.code() {
-            Some(exit_code) => Self::ChildFailure { exit_code },
-            None => Self::ChildKilled {
+            Some(exit_code) => ChildFailureSnafu { exit_code }.build(),
+            None => ChildKilledSnafu {
                 signal: status.signal().unwrap(),
-            },
+            }
+            .build(),
         }
     }
 }

src/nix/host/ssh.rs

@@ -398,7 +398,7 @@ impl Ssh {
         match self.run_command(self.ssh(&["reboot"])).await {
             Ok(()) => Ok(()),
             Err(e) => {
-                if let ColmenaError::ChildFailure { exit_code: 255 } = e {
+                if let ColmenaError::ChildFailure { exit_code: 255, .. } = e {
                     // Assume it's "Connection closed by remote host"
                     Ok(())
                 } else {

src/troubleshooter.rs

@@ -6,6 +6,7 @@ use std::env;
 use std::future::Future;
 
 use clap::{parser::ValueSource as ClapValueSource, ArgMatches};
+use snafu::ErrorCompat;
 
 use crate::error::ColmenaError;
 
@@ -61,5 +62,21 @@ fn troubleshoot(
         };
     }
 
+    if let Some(bt) = ErrorCompat::backtrace(error) {
+        if backtrace_enabled() {
+            eprintln!("Backtrace:");
+            eprint!("{:?}", bt);
+        } else {
+            eprintln!("Hint: Backtrace available - Use `RUST_BACKTRACE=1` environment variable to display a backtrace");
+        }
+    }
+
     Ok(())
 }
+
+fn backtrace_enabled() -> bool {
+    match env::var("RUST_BACKTRACE") {
+        Ok(backtrace_conf) => backtrace_conf != "0",
+        _ => false,
+    }
+}