Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix the trace payload size hint #664

Merged
merged 2 commits into from
Oct 3, 2024
+75 −54
Merged

Fix the trace payload size hint #664

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
80d69c1
Fix the trace payload size hint
bwoebi Oct 3, 2024
ce20078
Add test
bwoebi Oct 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions sidecar/src/service/sidecar_server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,17 +265,17 @@ impl SidecarServer {
}
};

let size = data.len();

match tracer_payload::TracerPayloadParams::new(
let mut size = 0;
let mut processor = tracer_payload::DefaultTraceChunkProcessor;
let mut payload_params = tracer_payload::TracerPayloadParams::new(
data,
&headers,
&mut tracer_payload::DefaultTraceChunkProcessor,
&mut processor,
target.api_key.is_some(),
TraceEncoding::V04,
)
.try_into()
{
);
payload_params.measure_size(&mut size);
match payload_params.try_into() {
Ok(payload) => {
let data = SendData::new(size, payload, headers, target, None);
self.trace_flusher.enqueue(data);
Expand Down
85 changes: 46 additions & 39 deletions trace-utils/src/msgpack_decoder/v04/decoder/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,49 +47,56 @@ use tinybytes::{Bytes, BytesString};
/// };
/// let encoded_data = to_vec_named(&vec![vec![span]]).unwrap();
/// let encoded_data_as_tinybytes = tinybytes::Bytes::from(encoded_data);
/// let decoded_traces = from_slice(encoded_data_as_tinybytes).expect("Decoding failed");
/// let (decoded_traces, _) = from_slice(encoded_data_as_tinybytes).expect("Decoding failed");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
/// let (decoded_traces, _) = from_slice(encoded_data_as_tinybytes).expect("Decoding failed");
/// let (decoded_traces, _payload_size) = from_slice(encoded_data_as_tinybytes).expect("Decoding failed");

I think it's helpful to at least name the variable, even if we don't use it in the example.

///
/// assert_eq!(1, decoded_traces.len());
/// assert_eq!(1, decoded_traces[0].len());
/// let decoded_span = &decoded_traces[0][0];
/// assert_eq!("test-span", decoded_span.name.as_str());
/// ```
pub fn from_slice(mut data: tinybytes::Bytes) -> Result<Vec<Vec<Span>>, DecodeError> {
pub fn from_slice(mut data: tinybytes::Bytes) -> Result<(Vec<Vec<Span>>, usize), DecodeError> {
let trace_count =
rmp::decode::read_array_len(unsafe { data.as_mut_slice() }).map_err(|_| {
DecodeError::InvalidFormat("Unable to read array len for trace count".to_owned())
})?;

(0..trace_count).try_fold(
Vec::with_capacity(
trace_count
.try_into()
.expect("Unable to cast trace_count to usize"),
),
|mut traces, _| {
let span_count =
rmp::decode::read_array_len(unsafe { data.as_mut_slice() }).map_err(|_| {
DecodeError::InvalidFormat("Unable to read array len for span count".to_owned())
})?;

let trace = (0..span_count).try_fold(
Vec::with_capacity(
span_count
.try_into()
.expect("Unable to cast span_count to usize"),
),
|mut trace, _| {
let span = decode_span(&mut data)?;
trace.push(span);
Ok(trace)
},
)?;

traces.push(trace);
let start_len = data.len();

Ok(traces)
},
)
Ok((
(0..trace_count).try_fold(
Vec::with_capacity(
trace_count
.try_into()
.expect("Unable to cast trace_count to usize"),
),
|mut traces, _| {
let span_count = rmp::decode::read_array_len(unsafe { data.as_mut_slice() })
.map_err(|_| {
DecodeError::InvalidFormat(
"Unable to read array len for span count".to_owned(),
)
})?;

let trace = (0..span_count).try_fold(
Vec::with_capacity(
span_count
.try_into()
.expect("Unable to cast span_count to usize"),
),
|mut trace, _| {
let span = decode_span(&mut data)?;
trace.push(span);
Ok(trace)
},
)?;

traces.push(trace);

Ok(traces)
},
)?,
start_len - data.len(),
))
}

#[inline]
Expand Down Expand Up @@ -270,7 +277,7 @@ mod tests {
..Default::default()
};
let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap();
let decoded_traces =
let (decoded_traces, _) =
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should probably have at least one unit test that checks we're returning the correct size.

from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed");

assert_eq!(1, decoded_traces.len());
Expand All @@ -290,7 +297,7 @@ mod tests {
span["meta_struct"] = json!(expected_meta_struct.clone());

let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap();
let decoded_traces =
let (decoded_traces, _) =
from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed");

assert_eq!(1, decoded_traces.len());
Expand All @@ -314,7 +321,7 @@ mod tests {
span["meta_struct"] = json!(expected_meta_struct.clone());

let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap();
let decoded_traces =
let (decoded_traces, _) =
from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed");

assert_eq!(1, decoded_traces.len());
Expand All @@ -340,7 +347,7 @@ mod tests {
span["meta"] = json!(expected_meta.clone());

let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap();
let decoded_traces =
let (decoded_traces, _) =
from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed");

assert_eq!(1, decoded_traces.len());
Expand Down Expand Up @@ -370,7 +377,7 @@ mod tests {
span["meta"] = json!(expected_meta.clone());

let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap();
let decoded_traces =
let (decoded_traces, _) =
from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed");

assert_eq!(1, decoded_traces.len());
Expand All @@ -392,7 +399,7 @@ mod tests {
let mut span = create_test_json_span(1, 2, 0, 0);
span["metrics"] = json!(expected_metrics.clone());
let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap();
let decoded_traces =
let (decoded_traces, _) =
from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed");

assert_eq!(1, decoded_traces.len());
Expand All @@ -416,7 +423,7 @@ mod tests {
let mut span = create_test_json_span(1, 2, 0, 0);
span["metrics"] = json!(expected_metrics.clone());
let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap();
let decoded_traces =
let (decoded_traces, _) =
from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed");

assert_eq!(1, decoded_traces.len());
Expand Down Expand Up @@ -449,7 +456,7 @@ mod tests {
span["span_links"] = json!([expected_span_link]);

let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap();
let decoded_traces =
let (decoded_traces, _) =
from_slice(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed");

assert_eq!(1, decoded_traces.len());
Expand Down
24 changes: 17 additions & 7 deletions trace-utils/src/tracer_payload.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,8 @@ pub struct TracerPayloadParams<'a, T: TraceChunkProcessor + 'a> {
data: tinybytes::Bytes,
/// Reference to `TracerHeaderTags` containing metadata for the trace.
tracer_header_tags: &'a TracerHeaderTags<'a>,
/// Amount of data consumed from buffer
size: Option<&'a mut usize>,
/// A mutable reference to an implementation of `TraceChunkProcessor` that processes each
/// `TraceChunk` after it is constructed but before it is added to the TracerPayloadCollection.
/// TraceChunks are only available for v07 traces.
Expand All @@ -194,11 +196,16 @@ impl<'a, T: TraceChunkProcessor + 'a> TracerPayloadParams<'a, T> {
TracerPayloadParams {
data,
tracer_header_tags,
size: None,
chunk_processor,
is_agentless,
encoding_type,
}
}

pub fn measure_size(&mut self, size: &'a mut usize) {
self.size = Some(size);
}
}
// TODO: APMSP-1282 - Implement TryInto for other encoding types. Supporting TraceChunkProcessor but
// not supporting v07 is a bit pointless for now.
Expand Down Expand Up @@ -253,13 +260,16 @@ impl<'a, T: TraceChunkProcessor + 'a> TryInto<TracerPayloadCollection>
fn try_into(self) -> Result<TracerPayloadCollection, Self::Error> {
match self.encoding_type {
TraceEncoding::V04 => {
let traces: Vec<Vec<Span>> =
match msgpack_decoder::v04::decoder::from_slice(self.data) {
Ok(res) => res,
Err(e) => {
anyhow::bail!("Error deserializing trace from request body: {e}")
}
};
let (traces, size) = match msgpack_decoder::v04::decoder::from_slice(self.data) {
Ok(res) => res,
Err(e) => {
anyhow::bail!("Error deserializing trace from request body: {e}")
}
};

if let Some(size_ref) = self.size {
*size_ref = size;
}

if traces.is_empty() {
anyhow::bail!("No traces deserialized from the request body.");
Expand Down
Loading