From c9c2d1fd91f6e27ee15996e65db5698cdb400eaa Mon Sep 17 00:00:00 2001 From: "Andres O. Vela" Date: Sun, 1 Oct 2023 12:07:49 +0200 Subject: [PATCH 01/19] Implement nested format parsing --- decoder/src/log/format.rs | 285 +++++++++++++++++++++++++++---- decoder/src/log/stdout_logger.rs | 1 + 2 files changed, 255 insertions(+), 31 deletions(-) diff --git a/decoder/src/log/format.rs b/decoder/src/log/format.rs index a63f4582..081f132b 100644 --- a/decoder/src/log/format.rs +++ b/decoder/src/log/format.rs @@ -4,8 +4,9 @@ use nom::{ character::complete::{char, digit1, one_of}, combinator::{map, map_res, opt}, multi::{many0, separated_list1}, - sequence::delimited, - IResult, Parser, + sequence::{delimited, preceded}, + error::{Error, ErrorKind, ParseError}, + Err, IResult, Parser, }; use std::str::FromStr; @@ -21,6 +22,7 @@ pub(super) enum LogMetadata { ModulePath, String(String), Timestamp, + NestedLogSegments(Vec), } #[derive(Debug, PartialEq, Clone, Copy)] @@ -63,6 +65,7 @@ enum IntermediateOutput { WidthAndAlignment((usize, Option)), Color(LogColor), Style(colored::Styles), + NestedLogSegment(LogSegment), } impl LogSegment { @@ -105,7 +108,58 @@ impl LogSegment { } } +/// This function is taken as-is from the parse-hyperlinks crate +/// https://docs.rs/parse-hyperlinks/0.9.3/src/parse_hyperlinks/lib.rs.html#24-68 +/// There is an open issue in nom to include this parser in nom 8.0 +/// https://github.com/rust-bakery/nom/issues/1253 +pub fn take_until_unbalanced( + opening_bracket: char, + closing_bracket: char, +) -> impl Fn(&str) -> IResult<&str, &str, ()> { + move |i: &str| { + let mut index = 0; + let mut bracket_counter = 0; + while let Some(n) = &i[index..].find(&[opening_bracket, closing_bracket, '\\'][..]) { + index += n; + let mut it = i[index..].chars(); + match it.next().unwrap_or_default() { + c if c == '\\' => { + // Skip the escape char `\`. + index += '\\'.len_utf8(); + // Skip also the following char. + let c = it.next().unwrap_or_default(); + index += c.len_utf8(); + } + c if c == opening_bracket => { + bracket_counter += 1; + index += opening_bracket.len_utf8(); + } + c if c == closing_bracket => { + // Closing bracket. + bracket_counter -= 1; + index += closing_bracket.len_utf8(); + } + // Can not happen. + _ => unreachable!(), + }; + // We found the unmatched closing bracket. + if bracket_counter == -1 { + // We do not consume it. + index -= closing_bracket.len_utf8(); + return Ok((&i[index..], &i[0..index])); + }; + } + + if bracket_counter == 0 { + Ok(("", i)) + } else { + Err(nom::Err::Error(())) + } + } +} + fn parse_metadata(input: &str) -> IResult<&str, IntermediateOutput, ()> { + println!("parse_metadata: \"{input}\""); let mut parse_type = map_res(take_while(char::is_alphabetic), move |s| { let metadata = match s { "f" => LogMetadata::FileName, @@ -124,6 +178,7 @@ fn parse_metadata(input: &str) -> IResult<&str, IntermediateOutput, ()> { } fn parse_color(input: &str) -> IResult<&str, IntermediateOutput, ()> { + println!("parse_color: \"{input}\""); let mut parse_type = map_res(take_while(char::is_alphabetic), move |s| { let color = match s { "severity" => LogColor::SeverityLevel, @@ -140,6 +195,7 @@ fn parse_color(input: &str) -> IResult<&str, IntermediateOutput, ()> { } fn parse_style(input: &str) -> IResult<&str, IntermediateOutput, ()> { + println!("parse_style: \"{input}\""); let mut parse_type = map_res(take_while(char::is_alphabetic), move |s| { let style = match s { "bold" => colored::Styles::Bold, @@ -156,6 +212,7 @@ fn parse_style(input: &str) -> IResult<&str, IntermediateOutput, ()> { } fn parse_width_and_alignment(input: &str) -> IResult<&str, IntermediateOutput, ()> { + println!("parse_width_and_alignment: \"{input}\""); let (input, alignment) = opt(map_res(one_of("<^>"), move |c| match c { '^' => Ok(Alignment::Center), '<' => Ok(Alignment::Left), @@ -171,44 +228,92 @@ fn parse_width_and_alignment(input: &str) -> IResult<&str, IntermediateOutput, ( )) } -fn parse_log_segment(input: &str) -> IResult<&str, LogSegment, ()> { - let (input, output) = separated_list1( - char(':'), - alt(( - parse_metadata, - parse_color, - parse_style, - parse_width_and_alignment, - )), - )(input)?; +fn parse_format(input: &str) -> IResult<&str, IntermediateOutput, ()> { + alt(( + parse_color, + parse_style, + parse_width_and_alignment, + )).parse(input) +} +fn build_log_segment(intermediate_output: Vec) -> Result> { + println!("\nbuild_log_segment ({NEST}): {:?}", intermediate_output); let mut metadata = None; let mut color = None; let mut style = None; let mut width_and_alignment = None; - for item in output { + let mut nested_segments = None; + for item in intermediate_output { match item { - IntermediateOutput::Metadata(m) if metadata.is_none() => metadata = Some(m), - IntermediateOutput::Color(c) if color.is_none() => color = Some(c), + IntermediateOutput::Metadata(m) if metadata.is_none() => { + println!("Found metadata"); + metadata = Some(m) + }, + IntermediateOutput::Color(c) if color.is_none() => { + println!("Found color"); + color = Some(c) + }, IntermediateOutput::Style(s) => { let mut styles: Vec = style.unwrap_or_default(); // A format with repeated style specifiers is not valid if styles.contains(&s) { + println!("Style fail"); return Err(nom::Err::Failure(())); } + println!("Found style"); styles.push(s); style = Some(styles); } IntermediateOutput::WidthAndAlignment(w) if width_and_alignment.is_none() => { + println!("Found width and alignment"); width_and_alignment = Some(w) } - _ => return Err(nom::Err::Failure(())), + IntermediateOutput::NestedLogSegment(s) => { + let mut segments: Vec = nested_segments.unwrap_or_default(); + + println!("Found nested"); + segments.push(s); + nested_segments = Some(segments); + } + _ => { + println!("Intermediate output fail"); + return Err(nom::Err::Failure(())) + }, + } + } + + if NEST { + let Some(nested_segments) = nested_segments else { + println!("Nested segments fail"); + return Err(nom::Err::Failure(())); + }; + + println!("Adding nested log segments to metadata for nested segment"); + metadata = Some(LogMetadata::NestedLogSegments(nested_segments)); + } else { + // We either have nested segments, or a metadata specifier, we can't have both + // This means we either have: + // metadata specifier: {L:underline} + // nested segments specifier: {[{L:<5:bold}]%underline} + if metadata.is_some() && nested_segments.is_some() { + return Err(nom::Err::Failure(())); + } + + // If we have a nested segment there must be exactly one, + // otherwise there's something weird going on + if let Some(segments) = nested_segments { + if segments.iter().count() == 1 { + return Ok(segments[0].clone()); + } else { + return Err(nom::Err::Failure(())); + } } } let Some(metadata) = metadata else { + println!("Metadata fail"); return Err(nom::Err::Failure(())); }; @@ -216,7 +321,7 @@ fn parse_log_segment(input: &str) -> IResult<&str, LogSegment, ()> { .map(|(w, a)| (Some(w), a)) .unwrap_or((None, None)); - let log_segment = LogSegment { + Ok(LogSegment { metadata, format: LogFormat { color, @@ -224,25 +329,90 @@ fn parse_log_segment(input: &str) -> IResult<&str, LogSegment, ()> { width, alignment, }, - }; + }) +} + +fn parse_log_segment(input: &str) -> IResult<&str, LogSegment, ()> { + println!("parse_log_segment ({NEST}): \"{input}\""); + + let (input, output) = if !NEST { + separated_list1( + char(':'), + alt(( + parse_metadata, + parse_format, + parse_nested::, + )), + )(input) + } else { + let parse_nested_argument = separated_list1( + char(':'), + alt((parse_metadata, parse_format)), + ); + + let parse_nested_log_segment = map_res(parse_nested_argument, |result| { + let log_segment = build_log_segment::(result)?; + Ok::>(IntermediateOutput::NestedLogSegment(log_segment)) + }); + + separated_list1( + char('%'), + alt(( + parse_nested_log_segment, + parse_format, + parse_nested::, + )), + )(input) + }?; + + let log_segment = build_log_segment::(output)?; Ok((input, log_segment)) } -fn parse_argument(input: &str) -> IResult<&str, LogSegment, ()> { - let mut parse_enclosed = delimited(char('{'), parse_log_segment, char('}')); - parse_enclosed.parse(input) +fn parse_argument(input: &str) -> IResult<&str, LogSegment, ()> { + println!("parse_argument ({NEST}): \"{input}\""); + let take_between_matching_brackets = delimited( + char('{'), + take_until_unbalanced('{', '}'), + char('}') + ); + + take_between_matching_brackets.and_then(parse_log_segment::).parse(input) } -fn parse_string_segment(input: &str) -> IResult<&str, LogSegment, ()> { - map(take_till1(|c| c == '{'), |s: &str| { +fn parse_string_segment(input: &str) -> IResult<&str, LogSegment, ()> { + println!("parse_string_segment: \"{input}\""); + map(take_till1(|c| { + if !NEST { + c == '{' + } else { + c == '{' || c == '%' + } + }), |s: &str| { LogSegment::new(LogMetadata::String(s.to_string())) }) .parse(input) } +fn parse_nested(input: &str) -> IResult<&str, IntermediateOutput, ()> { + println!("\nIN parse_nested ({NEST}): \"{input}\" ----------------"); + + let parse_nested_argument = map_res(parse_argument::, |result| Ok::>(IntermediateOutput::NestedLogSegment(result))); + let parse_nested_string_segment = map_res(parse_string_segment::, |result| Ok::>(IntermediateOutput::NestedLogSegment(result))); + let parse_nested_format = preceded(char('%'), parse_format); + let mut parse_all = many0(alt((parse_nested_argument, parse_nested_format, parse_nested_string_segment))); + + let (new_input, output) = parse_all(input)?; + + let log_segment = build_log_segment::(output)?; + + println!("\nOUT parse_nested ({NEST}): \"{new_input}\" ---------------"); + Ok((new_input, IntermediateOutput::NestedLogSegment(log_segment))) +} + pub(super) fn parse(input: &str) -> Result, String> { - let mut parse_all = many0(alt((parse_argument, parse_string_segment))); + let mut parse_all = many0(alt((parse_argument::, parse_string_segment::))); parse_all(input) .map(|(_, output)| output) @@ -277,7 +447,7 @@ mod tests { #[test] fn test_parse_string_segment() { - let result = parse_string_segment("Log: {t}"); + let result = parse_string_segment::("Log: {t}"); let (input, output) = result.unwrap(); assert_eq!(input, "{t}"); assert_eq!( @@ -288,19 +458,19 @@ mod tests { #[test] fn test_parse_empty_string_segment() { - let result = parse_string_segment(""); + let result = parse_string_segment::(""); assert!(result.is_err()); } #[test] fn test_parse_timestamp_argument() { - let result = parse_argument("{t}"); + let result = parse_argument::("{t}"); assert_eq!(result, Ok(("", LogSegment::new(LogMetadata::Timestamp)))); } #[test] fn test_parse_argument_with_color() { - let result = parse_log_segment("t:werror"); + let result = parse_log_segment::("t:werror"); let expected_output = LogSegment::new(LogMetadata::Timestamp).with_color(LogColor::WarnError); assert_eq!(result, Ok(("", expected_output))); @@ -308,7 +478,7 @@ mod tests { #[test] fn test_parse_argument_with_extra_format_parameters_width_first() { - let result = parse_argument("{t:>8:white}"); + let result = parse_argument::("{t:>8:white}"); let expected_output = LogSegment::new(LogMetadata::Timestamp) .with_width(8) .with_alignment(Alignment::Right) @@ -318,7 +488,7 @@ mod tests { #[test] fn test_parse_argument_with_extra_format_parameters_color_first() { - let result = parse_argument("{f:werror:<25}"); + let result = parse_argument::("{f:werror:<25}"); let expected_output = LogSegment::new(LogMetadata::FileName) .with_width(25) .with_alignment(Alignment::Left) @@ -328,7 +498,7 @@ mod tests { #[test] fn test_parse_invalid_argument() { - let result = parse_argument("{foo}"); + let result = parse_argument::("{foo}"); assert_eq!(result, Result::Err(nom::Err::Error(()))); } @@ -414,4 +584,57 @@ mod tests { let result = parse(log_template); assert!(result.is_err()); } + + #[test] + fn test_parse_single_nested_format() { + let log_template = "{[{L:<5:bold}]%underline%italic} {s}"; + let expected_output = vec![ + LogSegment::new(LogMetadata::NestedLogSegments( + vec![ + LogSegment::new(LogMetadata::String("[".to_string())), + LogSegment::new(LogMetadata::LogLevel) + .with_alignment(Alignment::Left) + .with_width(5) + .with_style(colored::Styles::Bold), + LogSegment::new(LogMetadata::String("]".to_string())), + ] + )) + .with_style(colored::Styles::Underline) + .with_style(colored::Styles::Italic), + LogSegment::new(LogMetadata::String(" ".to_string())), + LogSegment::new(LogMetadata::Log), + ]; + let result = parse(log_template); + assert_eq!(result, Ok(expected_output)); + } + + + /// Note: A format string with a bad format specifier doesn't actually make the parser fail. + /// It will just "eat" the bad specifier and will output a Log + #[test] + fn test_parse_single_nested_format_with_bad_specifier() { + let log_template = "{[{L:<5:bold}]%bad%underline} {s}"; + let expected_output = vec![ + LogSegment::new(LogMetadata::NestedLogSegments( + vec![ + LogSegment::new(LogMetadata::String("[".to_string())), + LogSegment::new(LogMetadata::LogLevel) + .with_alignment(Alignment::Left) + .with_width(5) + .with_style(colored::Styles::Bold), + LogSegment::new(LogMetadata::String("]".to_string())), + ] + )).with_style(colored::Styles::Underline), + LogSegment::new(LogMetadata::String(" ".to_string())), + LogSegment::new(LogMetadata::Log), + ]; + let result = parse(log_template); + assert_eq!(result, Ok(expected_output)); + } + + #[test] + fn test_parse_double_nested_format() { + let log_template = "{{[{L:<5}]%bold} {f:>20}:%<30} {s}"; + assert!(true); + } } diff --git a/decoder/src/log/stdout_logger.rs b/decoder/src/log/stdout_logger.rs index 1e3ff12c..53578f5d 100644 --- a/decoder/src/log/stdout_logger.rs +++ b/decoder/src/log/stdout_logger.rs @@ -164,6 +164,7 @@ impl<'a> Printer<'a> { LogMetadata::LineNumber => self.print_line_number(sink, &segment.format), LogMetadata::LogLevel => self.print_log_level(sink, &segment.format), LogMetadata::Log => self.print_log(sink, &segment.format), + _ => return Ok(()) }?; } writeln!(sink) From 093ebc805fb4372b6dd8d50d57125ec6a9ac9851 Mon Sep 17 00:00:00 2001 From: "Andres O. Vela" Date: Sun, 1 Oct 2023 15:41:25 +0200 Subject: [PATCH 02/19] Fix issue with bad specifiers on nested format --- decoder/src/log/format.rs | 68 +++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/decoder/src/log/format.rs b/decoder/src/log/format.rs index 081f132b..13a88f27 100644 --- a/decoder/src/log/format.rs +++ b/decoder/src/log/format.rs @@ -25,6 +25,17 @@ pub(super) enum LogMetadata { NestedLogSegments(Vec), } +impl LogMetadata { + /// Checks whether this `LogMetadata` came from a specifier such as + /// {t}, {f}, etc. + fn is_metadata_specifier(&self) -> bool { + match self { + LogMetadata::String(_) | LogMetadata::NestedLogSegments(_) => false, + _ => true, + } + } +} + #[derive(Debug, PartialEq, Clone, Copy)] pub(super) enum LogColor { /// User-defined color @@ -228,12 +239,21 @@ fn parse_width_and_alignment(input: &str) -> IResult<&str, IntermediateOutput, ( )) } -fn parse_format(input: &str) -> IResult<&str, IntermediateOutput, ()> { - alt(( +fn parse_format(input: &str) -> IResult<&str, IntermediateOutput, ()> { + let result = alt(( parse_color, parse_style, parse_width_and_alignment, - )).parse(input) + )).parse(input); + + if !FAIL_ON_ERR { + result + } else { + match result { + Ok(r) => Ok(r), + Err(_) => Err(nom::Err::Failure(())), + } + } } fn build_log_segment(intermediate_output: Vec) -> Result> { @@ -290,6 +310,13 @@ fn build_log_segment(intermediate_output: Vec(intermediate_output: Vec(input: &str) -> IResult<&str, LogSegment, ()> { - println!("parse_log_segment ({NEST}): \"{input}\""); + println!("IN parse_log_segment ({NEST}): \"{input}\""); let (input, output) = if !NEST { separated_list1( char(':'), alt(( parse_metadata, - parse_format, + parse_format::, parse_nested::, )), )(input) } else { let parse_nested_argument = separated_list1( char(':'), - alt((parse_metadata, parse_format)), + alt((parse_metadata, parse_format::)), ); let parse_nested_log_segment = map_res(parse_nested_argument, |result| { @@ -360,12 +387,14 @@ fn parse_log_segment(input: &str) -> IResult<&str, LogSegment, char('%'), alt(( parse_nested_log_segment, - parse_format, + parse_format::, parse_nested::, )), )(input) }?; + println!("OUT parse_log_segment ({NEST}): \"{input}\""); + let log_segment = build_log_segment::(output)?; Ok((input, log_segment)) } @@ -400,8 +429,8 @@ fn parse_nested(input: &str) -> IResult<&str, IntermediateOutp let parse_nested_argument = map_res(parse_argument::, |result| Ok::>(IntermediateOutput::NestedLogSegment(result))); let parse_nested_string_segment = map_res(parse_string_segment::, |result| Ok::>(IntermediateOutput::NestedLogSegment(result))); - let parse_nested_format = preceded(char('%'), parse_format); - let mut parse_all = many0(alt((parse_nested_argument, parse_nested_format, parse_nested_string_segment))); + let parse_nested_format = preceded(char('%'), parse_format::); + let mut parse_all = many0(alt((parse_nested_argument, parse_nested_string_segment, parse_nested_format))); let (new_input, output) = parse_all(input)?; @@ -499,7 +528,7 @@ mod tests { #[test] fn test_parse_invalid_argument() { let result = parse_argument::("{foo}"); - assert_eq!(result, Result::Err(nom::Err::Error(()))); + assert!(result.is_err()); } #[test] @@ -608,28 +637,11 @@ mod tests { assert_eq!(result, Ok(expected_output)); } - - /// Note: A format string with a bad format specifier doesn't actually make the parser fail. - /// It will just "eat" the bad specifier and will output a Log #[test] fn test_parse_single_nested_format_with_bad_specifier() { let log_template = "{[{L:<5:bold}]%bad%underline} {s}"; - let expected_output = vec![ - LogSegment::new(LogMetadata::NestedLogSegments( - vec![ - LogSegment::new(LogMetadata::String("[".to_string())), - LogSegment::new(LogMetadata::LogLevel) - .with_alignment(Alignment::Left) - .with_width(5) - .with_style(colored::Styles::Bold), - LogSegment::new(LogMetadata::String("]".to_string())), - ] - )).with_style(colored::Styles::Underline), - LogSegment::new(LogMetadata::String(" ".to_string())), - LogSegment::new(LogMetadata::Log), - ]; let result = parse(log_template); - assert_eq!(result, Ok(expected_output)); + assert!(result.is_err()); } #[test] From 05635769f3f4acafe0c1fa0cfb271e993e4484c0 Mon Sep 17 00:00:00 2001 From: "Andres O. Vela" Date: Sun, 1 Oct 2023 16:22:32 +0200 Subject: [PATCH 03/19] Implement support for doubly nested formats --- decoder/src/log/format.rs | 45 ++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/decoder/src/log/format.rs b/decoder/src/log/format.rs index 13a88f27..4db58082 100644 --- a/decoder/src/log/format.rs +++ b/decoder/src/log/format.rs @@ -410,15 +410,9 @@ fn parse_argument(input: &str) -> IResult<&str, LogSegment, () take_between_matching_brackets.and_then(parse_log_segment::).parse(input) } -fn parse_string_segment(input: &str) -> IResult<&str, LogSegment, ()> { +fn parse_string_segment(input: &str) -> IResult<&str, LogSegment, ()> { println!("parse_string_segment: \"{input}\""); - map(take_till1(|c| { - if !NEST { - c == '{' - } else { - c == '{' || c == '%' - } - }), |s: &str| { + map(take_till1(|c| c == '{' || c == '%'), |s: &str| { LogSegment::new(LogMetadata::String(s.to_string())) }) .parse(input) @@ -428,7 +422,7 @@ fn parse_nested(input: &str) -> IResult<&str, IntermediateOutp println!("\nIN parse_nested ({NEST}): \"{input}\" ----------------"); let parse_nested_argument = map_res(parse_argument::, |result| Ok::>(IntermediateOutput::NestedLogSegment(result))); - let parse_nested_string_segment = map_res(parse_string_segment::, |result| Ok::>(IntermediateOutput::NestedLogSegment(result))); + let parse_nested_string_segment = map_res(parse_string_segment, |result| Ok::>(IntermediateOutput::NestedLogSegment(result))); let parse_nested_format = preceded(char('%'), parse_format::); let mut parse_all = many0(alt((parse_nested_argument, parse_nested_string_segment, parse_nested_format))); @@ -441,7 +435,7 @@ fn parse_nested(input: &str) -> IResult<&str, IntermediateOutp } pub(super) fn parse(input: &str) -> Result, String> { - let mut parse_all = many0(alt((parse_argument::, parse_string_segment::))); + let mut parse_all = many0(alt((parse_argument::, parse_string_segment))); parse_all(input) .map(|(_, output)| output) @@ -476,7 +470,7 @@ mod tests { #[test] fn test_parse_string_segment() { - let result = parse_string_segment::("Log: {t}"); + let result = parse_string_segment("Log: {t}"); let (input, output) = result.unwrap(); assert_eq!(input, "{t}"); assert_eq!( @@ -487,7 +481,7 @@ mod tests { #[test] fn test_parse_empty_string_segment() { - let result = parse_string_segment::(""); + let result = parse_string_segment(""); assert!(result.is_err()); } @@ -647,6 +641,31 @@ mod tests { #[test] fn test_parse_double_nested_format() { let log_template = "{{[{L:<5}]%bold} {f:>20}:%<30} {s}"; - assert!(true); + let expected_output = vec![ + LogSegment::new(LogMetadata::NestedLogSegments( + vec![ + LogSegment::new(LogMetadata::NestedLogSegments( + vec![ + LogSegment::new(LogMetadata::String("[".to_string())), + LogSegment::new(LogMetadata::LogLevel) + .with_alignment(Alignment::Left) + .with_width(5), + LogSegment::new(LogMetadata::String("]".to_string())), + ] + )).with_style(colored::Styles::Bold), + LogSegment::new(LogMetadata::String(" ".to_string())), + LogSegment::new(LogMetadata::FileName) + .with_alignment(Alignment::Right) + .with_width(20), + LogSegment::new(LogMetadata::String(":".to_string())), + ] + )) + .with_alignment(Alignment::Left) + .with_width(30), + LogSegment::new(LogMetadata::String(" ".to_string())), + LogSegment::new(LogMetadata::Log), + ]; + let result = parse(log_template); + assert_eq!(result, Ok(expected_output)); } } From cb1cd7d5281ea17bc18d38f61ab424eb66551403 Mon Sep 17 00:00:00 2001 From: "Andres O. Vela" Date: Sun, 1 Oct 2023 16:24:39 +0200 Subject: [PATCH 04/19] Remove debug prints --- decoder/src/log/format.rs | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/decoder/src/log/format.rs b/decoder/src/log/format.rs index 4db58082..a5588375 100644 --- a/decoder/src/log/format.rs +++ b/decoder/src/log/format.rs @@ -170,7 +170,6 @@ pub fn take_until_unbalanced( } fn parse_metadata(input: &str) -> IResult<&str, IntermediateOutput, ()> { - println!("parse_metadata: \"{input}\""); let mut parse_type = map_res(take_while(char::is_alphabetic), move |s| { let metadata = match s { "f" => LogMetadata::FileName, @@ -189,7 +188,6 @@ fn parse_metadata(input: &str) -> IResult<&str, IntermediateOutput, ()> { } fn parse_color(input: &str) -> IResult<&str, IntermediateOutput, ()> { - println!("parse_color: \"{input}\""); let mut parse_type = map_res(take_while(char::is_alphabetic), move |s| { let color = match s { "severity" => LogColor::SeverityLevel, @@ -206,7 +204,6 @@ fn parse_color(input: &str) -> IResult<&str, IntermediateOutput, ()> { } fn parse_style(input: &str) -> IResult<&str, IntermediateOutput, ()> { - println!("parse_style: \"{input}\""); let mut parse_type = map_res(take_while(char::is_alphabetic), move |s| { let style = match s { "bold" => colored::Styles::Bold, @@ -223,7 +220,6 @@ fn parse_style(input: &str) -> IResult<&str, IntermediateOutput, ()> { } fn parse_width_and_alignment(input: &str) -> IResult<&str, IntermediateOutput, ()> { - println!("parse_width_and_alignment: \"{input}\""); let (input, alignment) = opt(map_res(one_of("<^>"), move |c| match c { '^' => Ok(Alignment::Center), '<' => Ok(Alignment::Left), @@ -257,7 +253,6 @@ fn parse_format(input: &str) -> IResult<&str, Intermedi } fn build_log_segment(intermediate_output: Vec) -> Result> { - println!("\nbuild_log_segment ({NEST}): {:?}", intermediate_output); let mut metadata = None; let mut color = None; let mut style = None; @@ -266,11 +261,9 @@ fn build_log_segment(intermediate_output: Vec { - println!("Found metadata"); metadata = Some(m) }, IntermediateOutput::Color(c) if color.is_none() => { - println!("Found color"); color = Some(c) }, IntermediateOutput::Style(s) => { @@ -278,27 +271,21 @@ fn build_log_segment(intermediate_output: Vec { - println!("Found width and alignment"); width_and_alignment = Some(w) } IntermediateOutput::NestedLogSegment(s) => { let mut segments: Vec = nested_segments.unwrap_or_default(); - - println!("Found nested"); segments.push(s); nested_segments = Some(segments); } _ => { - println!("Intermediate output fail"); return Err(nom::Err::Failure(())) }, } @@ -306,7 +293,6 @@ fn build_log_segment(intermediate_output: Vec(intermediate_output: Vec(intermediate_output: Vec(intermediate_output: Vec(input: &str) -> IResult<&str, LogSegment, ()> { - println!("IN parse_log_segment ({NEST}): \"{input}\""); - let (input, output) = if !NEST { separated_list1( char(':'), @@ -393,14 +374,11 @@ fn parse_log_segment(input: &str) -> IResult<&str, LogSegment, )(input) }?; - println!("OUT parse_log_segment ({NEST}): \"{input}\""); - let log_segment = build_log_segment::(output)?; Ok((input, log_segment)) } fn parse_argument(input: &str) -> IResult<&str, LogSegment, ()> { - println!("parse_argument ({NEST}): \"{input}\""); let take_between_matching_brackets = delimited( char('{'), take_until_unbalanced('{', '}'), @@ -411,7 +389,6 @@ fn parse_argument(input: &str) -> IResult<&str, LogSegment, () } fn parse_string_segment(input: &str) -> IResult<&str, LogSegment, ()> { - println!("parse_string_segment: \"{input}\""); map(take_till1(|c| c == '{' || c == '%'), |s: &str| { LogSegment::new(LogMetadata::String(s.to_string())) }) @@ -419,18 +396,13 @@ fn parse_string_segment(input: &str) -> IResult<&str, LogSegment, ()> { } fn parse_nested(input: &str) -> IResult<&str, IntermediateOutput, ()> { - println!("\nIN parse_nested ({NEST}): \"{input}\" ----------------"); - let parse_nested_argument = map_res(parse_argument::, |result| Ok::>(IntermediateOutput::NestedLogSegment(result))); let parse_nested_string_segment = map_res(parse_string_segment, |result| Ok::>(IntermediateOutput::NestedLogSegment(result))); let parse_nested_format = preceded(char('%'), parse_format::); let mut parse_all = many0(alt((parse_nested_argument, parse_nested_string_segment, parse_nested_format))); let (new_input, output) = parse_all(input)?; - let log_segment = build_log_segment::(output)?; - - println!("\nOUT parse_nested ({NEST}): \"{new_input}\" ---------------"); Ok((new_input, IntermediateOutput::NestedLogSegment(log_segment))) } From a0c784a87a79afe670dec3492551c07eff6bb130 Mon Sep 17 00:00:00 2001 From: "Andres O. Vela" Date: Sun, 1 Oct 2023 16:25:27 +0200 Subject: [PATCH 05/19] cargo fmt --- decoder/src/log/format.rs | 123 ++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 66 deletions(-) diff --git a/decoder/src/log/format.rs b/decoder/src/log/format.rs index a5588375..db654eb1 100644 --- a/decoder/src/log/format.rs +++ b/decoder/src/log/format.rs @@ -3,9 +3,9 @@ use nom::{ bytes::complete::{take_till1, take_while}, character::complete::{char, digit1, one_of}, combinator::{map, map_res, opt}, + error::{Error, ErrorKind, ParseError}, multi::{many0, separated_list1}, sequence::{delimited, preceded}, - error::{Error, ErrorKind, ParseError}, Err, IResult, Parser, }; @@ -236,11 +236,7 @@ fn parse_width_and_alignment(input: &str) -> IResult<&str, IntermediateOutput, ( } fn parse_format(input: &str) -> IResult<&str, IntermediateOutput, ()> { - let result = alt(( - parse_color, - parse_style, - parse_width_and_alignment, - )).parse(input); + let result = alt((parse_color, parse_style, parse_width_and_alignment)).parse(input); if !FAIL_ON_ERR { result @@ -252,7 +248,9 @@ fn parse_format(input: &str) -> IResult<&str, Intermedi } } -fn build_log_segment(intermediate_output: Vec) -> Result> { +fn build_log_segment( + intermediate_output: Vec, +) -> Result> { let mut metadata = None; let mut color = None; let mut style = None; @@ -260,12 +258,8 @@ fn build_log_segment(intermediate_output: Vec { - metadata = Some(m) - }, - IntermediateOutput::Color(c) if color.is_none() => { - color = Some(c) - }, + IntermediateOutput::Metadata(m) if metadata.is_none() => metadata = Some(m), + IntermediateOutput::Color(c) if color.is_none() => color = Some(c), IntermediateOutput::Style(s) => { let mut styles: Vec = style.unwrap_or_default(); @@ -285,9 +279,7 @@ fn build_log_segment(intermediate_output: Vec { - return Err(nom::Err::Failure(())) - }, + _ => return Err(nom::Err::Failure(())), } } @@ -298,7 +290,9 @@ fn build_log_segment(intermediate_output: Vec(intermediate_output: Vec(input: &str) -> IResult<&str, LogSegment, ()> { let (input, output) = if !NEST { separated_list1( char(':'), - alt(( - parse_metadata, - parse_format::, - parse_nested::, - )), + alt((parse_metadata, parse_format::, parse_nested::)), )(input) } else { - let parse_nested_argument = separated_list1( - char(':'), - alt((parse_metadata, parse_format::)), - ); + let parse_nested_argument = + separated_list1(char(':'), alt((parse_metadata, parse_format::))); let parse_nested_log_segment = map_res(parse_nested_argument, |result| { let log_segment = build_log_segment::(result)?; - Ok::>(IntermediateOutput::NestedLogSegment(log_segment)) + Ok::>(IntermediateOutput::NestedLogSegment( + log_segment, + )) }); separated_list1( @@ -379,13 +368,12 @@ fn parse_log_segment(input: &str) -> IResult<&str, LogSegment, } fn parse_argument(input: &str) -> IResult<&str, LogSegment, ()> { - let take_between_matching_brackets = delimited( - char('{'), - take_until_unbalanced('{', '}'), - char('}') - ); - - take_between_matching_brackets.and_then(parse_log_segment::).parse(input) + let take_between_matching_brackets = + delimited(char('{'), take_until_unbalanced('{', '}'), char('}')); + + take_between_matching_brackets + .and_then(parse_log_segment::) + .parse(input) } fn parse_string_segment(input: &str) -> IResult<&str, LogSegment, ()> { @@ -396,10 +384,18 @@ fn parse_string_segment(input: &str) -> IResult<&str, LogSegment, ()> { } fn parse_nested(input: &str) -> IResult<&str, IntermediateOutput, ()> { - let parse_nested_argument = map_res(parse_argument::, |result| Ok::>(IntermediateOutput::NestedLogSegment(result))); - let parse_nested_string_segment = map_res(parse_string_segment, |result| Ok::>(IntermediateOutput::NestedLogSegment(result))); + let parse_nested_argument = map_res(parse_argument::, |result| { + Ok::>(IntermediateOutput::NestedLogSegment(result)) + }); + let parse_nested_string_segment = map_res(parse_string_segment, |result| { + Ok::>(IntermediateOutput::NestedLogSegment(result)) + }); let parse_nested_format = preceded(char('%'), parse_format::); - let mut parse_all = many0(alt((parse_nested_argument, parse_nested_string_segment, parse_nested_format))); + let mut parse_all = many0(alt(( + parse_nested_argument, + parse_nested_string_segment, + parse_nested_format, + ))); let (new_input, output) = parse_all(input)?; let log_segment = build_log_segment::(output)?; @@ -584,16 +580,14 @@ mod tests { fn test_parse_single_nested_format() { let log_template = "{[{L:<5:bold}]%underline%italic} {s}"; let expected_output = vec![ - LogSegment::new(LogMetadata::NestedLogSegments( - vec![ - LogSegment::new(LogMetadata::String("[".to_string())), - LogSegment::new(LogMetadata::LogLevel) - .with_alignment(Alignment::Left) - .with_width(5) - .with_style(colored::Styles::Bold), - LogSegment::new(LogMetadata::String("]".to_string())), - ] - )) + LogSegment::new(LogMetadata::NestedLogSegments(vec![ + LogSegment::new(LogMetadata::String("[".to_string())), + LogSegment::new(LogMetadata::LogLevel) + .with_alignment(Alignment::Left) + .with_width(5) + .with_style(colored::Styles::Bold), + LogSegment::new(LogMetadata::String("]".to_string())), + ])) .with_style(colored::Styles::Underline) .with_style(colored::Styles::Italic), LogSegment::new(LogMetadata::String(" ".to_string())), @@ -614,24 +608,21 @@ mod tests { fn test_parse_double_nested_format() { let log_template = "{{[{L:<5}]%bold} {f:>20}:%<30} {s}"; let expected_output = vec![ - LogSegment::new(LogMetadata::NestedLogSegments( - vec![ - LogSegment::new(LogMetadata::NestedLogSegments( - vec![ - LogSegment::new(LogMetadata::String("[".to_string())), - LogSegment::new(LogMetadata::LogLevel) - .with_alignment(Alignment::Left) - .with_width(5), - LogSegment::new(LogMetadata::String("]".to_string())), - ] - )).with_style(colored::Styles::Bold), - LogSegment::new(LogMetadata::String(" ".to_string())), - LogSegment::new(LogMetadata::FileName) - .with_alignment(Alignment::Right) - .with_width(20), - LogSegment::new(LogMetadata::String(":".to_string())), - ] - )) + LogSegment::new(LogMetadata::NestedLogSegments(vec![ + LogSegment::new(LogMetadata::NestedLogSegments(vec![ + LogSegment::new(LogMetadata::String("[".to_string())), + LogSegment::new(LogMetadata::LogLevel) + .with_alignment(Alignment::Left) + .with_width(5), + LogSegment::new(LogMetadata::String("]".to_string())), + ])) + .with_style(colored::Styles::Bold), + LogSegment::new(LogMetadata::String(" ".to_string())), + LogSegment::new(LogMetadata::FileName) + .with_alignment(Alignment::Right) + .with_width(20), + LogSegment::new(LogMetadata::String(":".to_string())), + ])) .with_alignment(Alignment::Left) .with_width(30), LogSegment::new(LogMetadata::String(" ".to_string())), From 40a964c6349bfa1f9fee0bdec6807dbc8793fe09 Mon Sep 17 00:00:00 2001 From: "Andres O. Vela" Date: Sun, 1 Oct 2023 16:27:31 +0200 Subject: [PATCH 06/19] cargo clippy --- decoder/src/log/format.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/decoder/src/log/format.rs b/decoder/src/log/format.rs index db654eb1..ebd4d570 100644 --- a/decoder/src/log/format.rs +++ b/decoder/src/log/format.rs @@ -3,10 +3,9 @@ use nom::{ bytes::complete::{take_till1, take_while}, character::complete::{char, digit1, one_of}, combinator::{map, map_res, opt}, - error::{Error, ErrorKind, ParseError}, multi::{many0, separated_list1}, sequence::{delimited, preceded}, - Err, IResult, Parser, + IResult, Parser, }; use std::str::FromStr; @@ -29,10 +28,10 @@ impl LogMetadata { /// Checks whether this `LogMetadata` came from a specifier such as /// {t}, {f}, etc. fn is_metadata_specifier(&self) -> bool { - match self { - LogMetadata::String(_) | LogMetadata::NestedLogSegments(_) => false, - _ => true, - } + !matches!( + self, + LogMetadata::String(_) | LogMetadata::NestedLogSegments(_) + ) } } @@ -309,7 +308,7 @@ fn build_log_segment( // If we have a nested segment there must be exactly one, // otherwise there's something weird going on if let Some(segments) = nested_segments { - if segments.iter().count() == 1 { + if segments.len() == 1 { return Ok(segments[0].clone()); } else { return Err(nom::Err::Failure(())); From f1c0544e649b7abe10234148bc4199172ec8046b Mon Sep 17 00:00:00 2001 From: "Andres O. Vela" Date: Sun, 1 Oct 2023 18:39:00 +0200 Subject: [PATCH 07/19] Implement support for printing nested formats --- decoder/src/log/stdout_logger.rs | 108 +++++++++++++++++-------------- 1 file changed, 59 insertions(+), 49 deletions(-) diff --git a/decoder/src/log/stdout_logger.rs b/decoder/src/log/stdout_logger.rs index 53578f5d..096c6096 100644 --- a/decoder/src/log/stdout_logger.rs +++ b/decoder/src/log/stdout_logger.rs @@ -147,6 +147,7 @@ impl<'a> Printer<'a> { } /// Pads the defmt timestamp to take up at least the given number of characters. + /// TODO: Remove this, shouldn't be needed now that we have width field support pub fn min_timestamp_width(&mut self, min_timestamp_width: usize) -> &mut Self { self.min_timestamp_width = min_timestamp_width; self @@ -155,42 +156,60 @@ impl<'a> Printer<'a> { /// Prints the formatted log frame to `sink`. pub fn print_frame(&self, sink: &mut W) -> io::Result<()> { for segment in self.format { - match &segment.metadata { - LogMetadata::String(s) => self.print_string(sink, s), - LogMetadata::Timestamp => self.print_timestamp(sink, &segment.format), - LogMetadata::FileName => self.print_file_name(sink, &segment.format), - LogMetadata::FilePath => self.print_file_path(sink, &segment.format), - LogMetadata::ModulePath => self.print_module_path(sink, &segment.format), - LogMetadata::LineNumber => self.print_line_number(sink, &segment.format), - LogMetadata::LogLevel => self.print_log_level(sink, &segment.format), - LogMetadata::Log => self.print_log(sink, &segment.format), - _ => return Ok(()) - }?; + let s = match &segment.metadata { + LogMetadata::String(s) => s.to_string(), + LogMetadata::Timestamp => self.build_timestamp(&segment.format), + LogMetadata::FileName => self.build_file_name(&segment.format), + LogMetadata::FilePath => self.build_file_path(&segment.format), + LogMetadata::ModulePath => self.build_module_path(&segment.format), + LogMetadata::LineNumber => self.build_line_number(&segment.format), + LogMetadata::LogLevel => self.build_log_level(&segment.format), + LogMetadata::Log => self.build_log(&segment.format), + LogMetadata::NestedLogSegments(segments) => self.build_nested(segments, &segment.format), + }; + + write!(sink, "{s}")?; } writeln!(sink) } - fn print_string(&self, sink: &mut W, s: &str) -> io::Result<()> { - write!(sink, "{s}") + fn build_nested(&self, segments: &[LogSegment], format: &LogFormat) -> String { + let mut result = String::new(); + for segment in segments { + let s = match &segment.metadata { + LogMetadata::String(s) => s.to_string(), + LogMetadata::Timestamp => self.build_timestamp(&segment.format), + LogMetadata::FileName => self.build_file_name(&segment.format), + LogMetadata::FilePath => self.build_file_path(&segment.format), + LogMetadata::ModulePath => self.build_module_path(&segment.format), + LogMetadata::LineNumber => self.build_line_number(&segment.format), + LogMetadata::LogLevel => self.build_log_level(&segment.format), + LogMetadata::Log => self.build_log(&segment.format), + LogMetadata::NestedLogSegments(segments) => self.build_nested(segments, &segment.format), + }; + result.push_str(&s); + } + + build_formatted_string(&result, format, 0, self.record_log_level(), format.color) } - fn print_timestamp(&self, sink: &mut W, format: &LogFormat) -> io::Result<()> { + fn build_timestamp(&self, format: &LogFormat) -> String { let s = match self.record { Record::Defmt(record) if !record.timestamp().is_empty() => record.timestamp(), _ => "