feat(main): Parse timestamps out of journald entries
Instead of relying on Stackdriver's ingestion timestamps, parse timestamps out of journal entries and provide those to Stackdriver. If a timestamp could not be parsed out of a log entry, the ingestion time is used as the fallback.
This commit is contained in:
parent
f626d88438
commit
87ab3c806c
2 changed files with 51 additions and 9 deletions
49
src/main.rs
49
src/main.rs
|
@ -40,12 +40,15 @@
|
||||||
#[macro_use] extern crate serde_json;
|
#[macro_use] extern crate serde_json;
|
||||||
#[macro_use] extern crate lazy_static;
|
#[macro_use] extern crate lazy_static;
|
||||||
|
|
||||||
extern crate failure;
|
extern crate chrono;
|
||||||
extern crate env_logger;
|
extern crate env_logger;
|
||||||
extern crate systemd;
|
extern crate failure;
|
||||||
extern crate serde;
|
|
||||||
extern crate reqwest;
|
extern crate reqwest;
|
||||||
|
extern crate serde;
|
||||||
|
extern crate systemd;
|
||||||
|
|
||||||
|
use chrono::offset::LocalResult;
|
||||||
|
use chrono::prelude::*;
|
||||||
use failure::ResultExt;
|
use failure::ResultExt;
|
||||||
use reqwest::{header, Client};
|
use reqwest::{header, Client};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
@ -174,12 +177,36 @@ fn message_to_payload(message: Option<String>) -> Payload {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Attempt to parse journald's microsecond timestamps into a UTC
|
||||||
|
/// timestamp.
|
||||||
|
///
|
||||||
|
/// Parse errors are dismissed and returned as empty options: There
|
||||||
|
/// simply aren't any useful fallback mechanisms other than defaulting
|
||||||
|
/// to ingestion time for journaldriver's use-case.
|
||||||
|
fn parse_microseconds(input: String) -> Option<DateTime<Utc>> {
|
||||||
|
if input.len() != 16 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let seconds: i64 = (&input[..10]).parse().ok()?;
|
||||||
|
let micros: u32 = (&input[10..]).parse().ok()?;
|
||||||
|
|
||||||
|
match Utc.timestamp_opt(seconds, micros * 1000) {
|
||||||
|
LocalResult::Single(time) => Some(time),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// This structure represents a log entry in the format expected by
|
/// This structure represents a log entry in the format expected by
|
||||||
/// the Stackdriver API.
|
/// the Stackdriver API.
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
struct LogEntry {
|
struct LogEntry {
|
||||||
labels: Value,
|
labels: Value,
|
||||||
|
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
timestamp: Option<DateTime<Utc>>,
|
||||||
|
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
payload: Payload,
|
payload: Payload,
|
||||||
}
|
}
|
||||||
|
@ -203,15 +230,19 @@ impl From<JournalRecord> for LogEntry {
|
||||||
// present on all others.
|
// present on all others.
|
||||||
let unit = record.remove("_SYSTEMD_UNIT");
|
let unit = record.remove("_SYSTEMD_UNIT");
|
||||||
|
|
||||||
// TODO: This timestamp (in microseconds) should be parsed
|
// The source timestamp (if present) is specified in
|
||||||
// into a DateTime<Utc> and used instead of the ingestion
|
// microseconds since epoch.
|
||||||
// time.
|
//
|
||||||
// let timestamp = record
|
// If it is not present or can not be parsed, journaldriver
|
||||||
// .remove("_SOURCE_REALTIME_TIMESTAMP")
|
// will not send a timestamp for the log entry and it will
|
||||||
// .map();
|
// default to the ingestion time.
|
||||||
|
let timestamp = record
|
||||||
|
.remove("_SOURCE_REALTIME_TIMESTAMP")
|
||||||
|
.and_then(parse_microseconds);
|
||||||
|
|
||||||
LogEntry {
|
LogEntry {
|
||||||
payload,
|
payload,
|
||||||
|
timestamp,
|
||||||
labels: json!({
|
labels: json!({
|
||||||
"host": hostname,
|
"host": hostname,
|
||||||
"unit": unit.unwrap_or_else(|| "syslog".into()),
|
"unit": unit.unwrap_or_else(|| "syslog".into()),
|
||||||
|
|
11
src/tests.rs
11
src/tests.rs
|
@ -5,6 +5,7 @@ use serde_json::to_string;
|
||||||
fn test_text_entry_serialization() {
|
fn test_text_entry_serialization() {
|
||||||
let entry = LogEntry {
|
let entry = LogEntry {
|
||||||
labels: Value::Null,
|
labels: Value::Null,
|
||||||
|
timestamp: None,
|
||||||
payload: Payload::TextPayload {
|
payload: Payload::TextPayload {
|
||||||
text_payload: "test entry".into(),
|
text_payload: "test entry".into(),
|
||||||
}
|
}
|
||||||
|
@ -20,6 +21,7 @@ fn test_text_entry_serialization() {
|
||||||
fn test_json_entry_serialization() {
|
fn test_json_entry_serialization() {
|
||||||
let entry = LogEntry {
|
let entry = LogEntry {
|
||||||
labels: Value::Null,
|
labels: Value::Null,
|
||||||
|
timestamp: None,
|
||||||
payload: Payload::TextPayload {
|
payload: Payload::TextPayload {
|
||||||
text_payload: "test entry".into(),
|
text_payload: "test entry".into(),
|
||||||
}
|
}
|
||||||
|
@ -78,3 +80,12 @@ fn test_json_no_object() {
|
||||||
|
|
||||||
assert_eq!(expected, payload, "Non-object JSON payload should be plain text");
|
assert_eq!(expected, payload, "Non-object JSON payload should be plain text");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_parse_microseconds() {
|
||||||
|
let input: String = "1529175149291187".into();
|
||||||
|
let expected: DateTime<Utc> = "2018-06-16T18:52:29.291187Z"
|
||||||
|
.to_string().parse().unwrap();
|
||||||
|
|
||||||
|
assert_eq!(Some(expected), parse_microseconds(input));
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue