From 0595669e6a9c0337bad6e6dc999e60c7ef9ccbbf Mon Sep 17 00:00:00 2001 From: Guillaume Binet Date: Thu, 30 May 2024 10:03:38 -0400 Subject: [PATCH] Logger serialization fixes. There are still an issue with Value and deserializing them with serde but serialization looks ok. --- copper/src/config.rs | 1 - copper_log/src/lib.rs | 1 - copper_log_reader/Cargo.toml | 3 +- copper_log_reader/src/lib.rs | 60 ++++++++++-------- .../test/copper_log_index/data.mdb | Bin 73728 -> 73728 bytes .../test/copper_log_index/lock.mdb | Bin 8192 -> 8192 bytes copper_log_reader/test/teststructlog.copper | Bin 5126 -> 5126 bytes copper_log_runtime/src/lib.rs | 45 +++++++++++-- copper_value/Cargo.toml | 4 +- copper_value/src/bdec.rs | 2 +- copper_value/src/benc.rs | 2 + copper_value/src/de.rs | 2 + copper_value/src/lib.rs | 2 +- copper_value/src/ser.rs | 47 ++++++++------ 14 files changed, 113 insertions(+), 56 deletions(-) diff --git a/copper/src/config.rs b/copper/src/config.rs index f12205ef4..5abf813b6 100644 --- a/copper/src/config.rs +++ b/copper/src/config.rs @@ -337,7 +337,6 @@ pub fn read_configuration(config_filename: &str) -> CuResult { // tests #[cfg(test)] mod tests { - use uom::si::power::watt; use uom::si::time::millisecond; use uom::si::time::second; diff --git a/copper_log/src/lib.rs b/copper_log/src/lib.rs index df21c74d5..49b616a30 100644 --- a/copper_log/src/lib.rs +++ b/copper_log/src/lib.rs @@ -5,7 +5,6 @@ extern crate proc_macro; use crate::index::{check_and_insert, intern_string}; use proc_macro::TokenStream; use quote::quote; -use std::fmt::Display; use syn::parse::Parser; use syn::Token; use syn::{Expr, ExprAssign, ExprLit, Lit}; diff --git a/copper_log_reader/Cargo.toml b/copper_log_reader/Cargo.toml index 918c5e39f..1477e43e4 100644 --- a/copper_log_reader/Cargo.toml +++ b/copper_log_reader/Cargo.toml @@ -7,4 +7,5 @@ edition = "2021" copper-log-runtime = { path = "../copper_log_runtime" } rkv = { version = "0.19.0", features = ["lmdb"] } byteorder = "1.5.0" -bincode = "2.0.0-rc.3" +bincode = { version = "2.0.0-rc.3", features = ["serde", "derive"] } +strfmt = "0.2.0" diff --git a/copper_log_reader/src/lib.rs b/copper_log_reader/src/lib.rs index a6c54199a..db2420efa 100644 --- a/copper_log_reader/src/lib.rs +++ b/copper_log_reader/src/lib.rs @@ -4,17 +4,19 @@ use byteorder::{ByteOrder, LittleEndian}; use copper_log_runtime::CuLogEntry; use rkv::backend::Lmdb; use rkv::{Rkv, StoreOptions, Writer}; +use std::collections::HashMap; use std::io::Read; use std::path::Path; +use strfmt::strfmt; -pub fn extract_copper_log_index(mut src: impl Read, index: &Path) { +pub fn extract_copper_log_with_index(mut src: impl Read, index: &Path) { let mut all_strings = Vec::::new(); let env = Rkv::new::(&index).unwrap(); let index_to_string = env .open_single("index_to_string", StoreOptions::default()) .expect("Failed to open index_to_string store"); - let db_eader = env.read().unwrap(); - let ri = index_to_string.iter_start(&db_eader); + let db_reader = env.read().unwrap(); + let ri = index_to_string.iter_start(&db_reader); let mut i = ri.expect("Failed to start iterator"); while let Some(Ok(v)) = i.next() { let (k, v) = v; @@ -29,14 +31,31 @@ pub fn extract_copper_log_index(mut src: impl Read, index: &Path) { println!("{} -> {}", index, s); } } - let entry = decode_from_std_read::(&mut src, standard()); - let entry = entry.expect("Failed to decode CuLogEntry"); - println!( - "Entry: {} -> {} with params {:?}", - entry.msg_index, all_strings[entry.msg_index as usize], entry.params - ); - //FIXME: Entry: 1 -> Just a string {} with params [I16(61)] - compile_error!("ici"); + loop { + let entry = decode_from_std_read::(&mut src, standard()); + if entry.is_err() { + break; + } + let entry = entry.unwrap(); + let mut format_string = all_strings[entry.msg_index as usize].clone(); + let mut vars = HashMap::new(); + + for (i, param) in entry.params.iter().enumerate() { + let param_as_string = format!("{}", param); + if entry.paramname_indexes[i] == 0 { + // Anonymous parameter + format_string = format_string.replacen("{}", ¶m_as_string, 1); + } else { + // Named parameter + let name = all_strings[entry.paramname_indexes[i] as usize].clone(); + vars.insert(name, param_as_string); + } + } + + // Use strfmt to replace named parameters + let result = strfmt(&format_string, &vars).unwrap(); + println!("Copper: {}", result); + } } #[cfg(test)] @@ -44,27 +63,16 @@ mod tests { use super::*; use std::io::Cursor; - /* - Just a string {} CuLogEntry { msg_index: 1, paramname_indexes: [0], params: [String("zarma")] } - anonymous param constants {} {} CuLogEntry { msg_index: 2, paramname_indexes: [0, 0], params: [U16(42), U8(43)] } - named param constants {} {} CuLogEntry { msg_index: 3, paramname_indexes: [4, 5], params: [I32(3), I32(2)] } - mixed named param constants, {} {} {} CuLogEntry { msg_index: 6, paramname_indexes: [0, 4, 5], params: [I32(54), I32(3), I32(2)] } - complex tuple CuLogEntry { msg_index: 7, paramname_indexes: [0], params: [Seq([I32(1), String("toto"), F64(3.34), Bool(true), Char('a')])] } - Struct CuLogEntry { msg_index: 8, paramname_indexes: [0], params: [Map({String("a"): I32(3), String("b"): I32(4)})] } - Allocations: +{}B -{}B CuLogEntry { msg_index: 1, paramname_indexes: [2, 3], params: [U64(1003932), U64(1002876)] } - AFTER CLOSE {} CuLogEntry { msg_index: 10, paramname_indexes: [0], params: [String("AFTER CLOSE")] } - */ - - // const stored_log: &[u8] = include_bytes!("../test/teststructlog.copper"); #[test] fn test_extract_copper_log() { - let hex_string = "01 01 00 01 05 7A 61 72 6D 61 02 02 00 00 02 2A 2B 03 02 04 05 02 06 04 06 03 00 04 05 03 6C 06 04 07 01 00 01 05 02 04 74 6F 74 6F B8 1E 85 EB 51 B8 0A 40 01 61 08 01 00 01 02 01 61 06 01 62 08 01 02 02 03 02 FC 9C 51 0F 00 FC 7C 4D 0F 00 0A 01 00 01 0B 41 46 54 45 52 20 43 4C 4F 53 45 43 4C 4F 53 45 00 00 00 00"; + let hex_string = "01 01 00 01 0C 05 7A 61 72 6D 61"; let bytes: Vec = hex_string .split_whitespace() .map(|s| u8::from_str_radix(s, 16).expect("Parse error")) .collect(); - let mut reader = Cursor::new(bytes.as_slice()); - extract_copper_log_index(reader, Path::new("test/copper_log_index")); + + let reader = Cursor::new(bytes.as_slice()); + extract_copper_log_with_index(reader, Path::new("test/copper_log_index")); } } diff --git a/copper_log_reader/test/copper_log_index/data.mdb b/copper_log_reader/test/copper_log_index/data.mdb index 5023d02911430d7d13560f973da9935e70a8c98b..1d870d537c11b5ac26bd0cbe5f5a5d07efacbee5 100644 GIT binary patch delta 1529 zcmbW0KWtJ_6vpp`XL%3Wd!UeFTP#Rol%=>C8=_1`9Z1}8@lWDFWAbd2)S-ioqf=jQ z;&X8|4x|JbaFZBgh|4344#vUMIB0~?$@k}el%ga?mcx7R-gCd-cfPyZ)ponu-en%P zuAEB{3w{YzQ!T4QMp!*Xt!u?ZY-&leZN+zJDbE_>BfX$)x}HnF$+u>*{mo4CoTg6p zHs#phkK<=Q=b(WV6TzT@G(O@pAfRlKNLL_qWrS!D3PprOL@20YoI5MBbY7qAHFAV# z@(L*O0;s4EqDB!)Nr@;at~#XziC4hRpPyG*@ZIUov9g!16=l}m+40`TZk~Cb85^TV z+aK&b{rlH5ddat)@T_dwraDs{h10Eg&FirZP1UP9dcOaK*#A;&--#^^0tg_000Iag zfB*srAb#YOo+C%kw%dD6E*4A9_^07S~4j+E7sdMMbmlmCK z`S3;O+;EG|oju^(YX_Wj@%*AOT@Jo}&W#lw#Q&0@MF0T=5I_I{1Q0*~0R#|essQo7 zW;yEdzv}Sa{K$3lIQ`T9V86Cs+1s{b|4RQze@P#wP5solZv+rP009ILKmY**5I_Kd zms{X1e|s_K9+K?FWFLL<#jVe?k3aqF?yc;bZ!>c~_VU?xm9bB6nozRdDtD_Zv)wL@ z?jCoG@pP*&Z}=xI<1_DCdcHLp?(XzwSurh}_hS#Af4||M7&9Eq`rBFF8rFCdGJ=KPdb8yA9(d=8Cyv@7Uk%_jZ^1 z|AzL!jL@A^JA+l?9q?$UIxyXrFu4-z&);W;I+k zK0MT4lLYFofIiGG7~^I^+#~b5(UJu??v-mIY6Q~Ib3z)Xr%y&qUvzf8FN3;@8ao|UZTqT@XFPL(*KE~{?PZD zIVAO;X!aoV@0fbpKT&;xuY?Gn8tea8`|ai35I_I{1Q0*~0R#|0009J+FR;4&e|>L2 zzY0N<2(^6!{VIe7=luGE1?SvsKz$v5s0V2Ix`9RK{7Qh@nDgn6@=EIem;cRUL_8{y(D8AbNrQ;(v+IB7gt_2q1s}0tg_000Ia!SK#FFzv}T7x1W~yU+WTx|C{^m=FSm7009IL zKmY**5I_I{1SD|s_&N=M)?ufB*srAbHESKeQF@?*yTnm*7rg2&ze cPvdAg!lC>Bxi<_Koj2WX+$FHvO|&85zoxqb7ytkO diff --git a/copper_log_reader/test/copper_log_index/lock.mdb b/copper_log_reader/test/copper_log_index/lock.mdb index 9e12e2a890036bcd21fe9d54b65e1a5e48f54bdf..827941176e894a54234d63d0fcb834e911708c13 100644 GIT binary patch delta 23 ecmZp0XmFTdz$h}&Fp))};&=7Nh4Jze6Sx6jZwOZa delta 23 ecmZp0XmFTdz$iP>Fp=d!ia_PYh4Jze6Sx6jBnTe> diff --git a/copper_log_reader/test/teststructlog.copper b/copper_log_reader/test/teststructlog.copper index b4a9b865e4c535fd43c7d1312111f659573cf1d9..ca6c1e1effae8501679e4a8a400d551144ce9ee3 100644 GIT binary patch delta 19 acmZqEXw%rhCBV+ZT9sIoo48T%D>DE%xCRse delta 109 zcmZqEXw%rhC6LHkl~|OU$i&3Jz@(+k%*4XV#KywL%)r9RoWsV#&d9*X%EVHVUy{E= zuJv``4lW1AL=K=J6JsJ9V-g1=6B9GjpE-g241a2T`5Cx?GTe@CA+A9R&OZLZu3%!L H;B009XKNNG diff --git a/copper_log_runtime/src/lib.rs b/copper_log_runtime/src/lib.rs index 1018d90da..e197dad62 100644 --- a/copper_log_runtime/src/lib.rs +++ b/copper_log_runtime/src/lib.rs @@ -1,10 +1,10 @@ -use bincode::enc::write::Writer; use bincode_derive::{Decode, Encode}; use copper_traits::{CuResult, Stream}; pub use copper_value as value; use copper_value::Value; use kanal::{bounded, Sender}; use once_cell::sync::OnceCell; +use serde::{Deserialize, Serialize}; use std::fmt::Display; use std::io::{stdout, Write}; use std::sync::{Arc, Mutex}; @@ -19,7 +19,7 @@ pub const ANONYMOUS: u32 = 0; /// The lifetime of this struct is the lifetime of the logger. pub struct LoggerRuntime {} -#[derive(Debug, Encode, Decode)] +#[derive(Debug, Encode, Decode, Serialize, Deserialize, PartialEq)] pub struct CuLogEntry { pub msg_index: u32, pub paramname_indexes: Vec, @@ -74,8 +74,6 @@ impl Drop for LoggerRuntime { fn initialize_queue(mut destination: impl Stream + 'static) -> Sender { let (sender, receiver) = bounded::(100); - let config = bincode::config::standard(); - let handle = thread::spawn(move || loop { if let Ok(data) = receiver.recv() { if let Err(err) = destination.log(&data) { @@ -110,3 +108,42 @@ pub fn log(entry: CuLogEntry) -> CuResult<()> { Err("Logger not initialized.".into()) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::CuLogEntry; + use bincode::config::standard; + use copper_value::Value; + + #[test] + fn test_encode_decode_structured_log() { + let log_entry = CuLogEntry { + msg_index: 1, + paramname_indexes: vec![2, 3], + params: vec![Value::String("test".to_string())], + }; + let encoded = bincode::encode_to_vec(&log_entry, standard()).unwrap(); + println!("{:?}", encoded); + let decoded_tuple: (CuLogEntry, usize) = + bincode::decode_from_slice(&encoded, standard()).unwrap(); + assert_eq!(log_entry, decoded_tuple.0); + } + + #[test] + #[ignore] // FIXME: the serde sere works but deser doesn't with the current implementation + fn test_serialize_deserialize_structured_log() { + // tests compatibility with std serde + let log_entry = CuLogEntry { + msg_index: 1, + paramname_indexes: vec![2, 3], + params: vec![Value::String("test".to_string())], + }; + let v = bincode::serde::encode_to_vec(&log_entry, standard()).unwrap(); + // Should be [1, 2, 2, 3, 1, 12, 4, 116, 101, 115, 116] + println!("{:?}", v); + let decoded: (CuLogEntry, usize) = + bincode::serde::decode_from_slice(v.as_slice(), standard()).unwrap(); + assert_eq!(log_entry, decoded.0); + } +} diff --git a/copper_value/Cargo.toml b/copper_value/Cargo.toml index a405b26be..577d46c43 100644 --- a/copper_value/Cargo.toml +++ b/copper_value/Cargo.toml @@ -12,7 +12,7 @@ license = "MIT" [dependencies] serde = "1.0.202" ordered-float = "4.2.0" -bincode = "2.0.0-rc.3" +bincode = { version = "2.0.0-rc.3", features = ["serde", "derive"] } [dev-dependencies] -serde_derive = "1.0.202" +serde_derive = "1.0.203" diff --git a/copper_value/src/bdec.rs b/copper_value/src/bdec.rs index 58722d9ba..11c7bb16c 100644 --- a/copper_value/src/bdec.rs +++ b/copper_value/src/bdec.rs @@ -29,7 +29,7 @@ impl Decode for Value { 16 => Ok(Value::Map(BTreeMap::::decode(decoder)?)), 17 => Ok(Value::Option(Option::>::decode(decoder)?)), 18 => Ok(Value::Newtype(Box::::decode(decoder)?)), - _ => Err(DecodeError::Other("Unknown Value variant")), + _ => Err(DecodeError::Other("Unknown Value varian")), } } } diff --git a/copper_value/src/benc.rs b/copper_value/src/benc.rs index a938f94a2..51d82a6c0 100644 --- a/copper_value/src/benc.rs +++ b/copper_value/src/benc.rs @@ -6,6 +6,8 @@ use bincode::error::EncodeError; impl Encode for Value { fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + let discriminant = self.discriminant() as u8; + discriminant.encode(encoder)?; match self { Value::U8(v) => v.to_owned().encode(encoder), Value::U16(v) => v.to_owned().encode(encoder), diff --git a/copper_value/src/de.rs b/copper_value/src/de.rs index ef51fd81c..1208f2f74 100644 --- a/copper_value/src/de.rs +++ b/copper_value/src/de.rs @@ -283,6 +283,7 @@ impl<'de> de::Visitor<'de> for ValueVisitor { } fn visit_newtype_struct>(self, d: D) -> Result { + println!("visit_newtype_struct"); d.deserialize_any(ValueVisitor) .map(|v| Value::Newtype(Box::new(v))) } @@ -314,6 +315,7 @@ impl<'de> de::Visitor<'de> for ValueVisitor { impl<'de> de::Deserialize<'de> for Value { fn deserialize>(d: D) -> Result { + println!("Value::deserialize"); d.deserialize_any(ValueVisitor) } } diff --git a/copper_value/src/lib.rs b/copper_value/src/lib.rs index 27b61a062..85c200304 100644 --- a/copper_value/src/lib.rs +++ b/copper_value/src/lib.rs @@ -36,7 +36,6 @@ pub enum Value { Char(char), String(String), - // InternedString(u32), Unit, Option(Option>), Newtype(Box), @@ -44,6 +43,7 @@ pub enum Value { Map(BTreeMap), Bytes(Vec), } + impl Display for Value { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { diff --git a/copper_value/src/ser.rs b/copper_value/src/ser.rs index 896175625..9562a2eeb 100644 --- a/copper_value/src/ser.rs +++ b/copper_value/src/ser.rs @@ -33,27 +33,36 @@ impl ser::Error for SerializerError { impl ser::Serialize for Value { fn serialize(&self, s: S) -> Result { match *self { - Value::Bool(v) => s.serialize_bool(v), - Value::U8(v) => s.serialize_u8(v), - Value::U16(v) => s.serialize_u16(v), - Value::U32(v) => s.serialize_u32(v), - Value::U64(v) => s.serialize_u64(v), - Value::I8(v) => s.serialize_i8(v), - Value::I16(v) => s.serialize_i16(v), - Value::I32(v) => s.serialize_i32(v), - Value::I64(v) => s.serialize_i64(v), - Value::F32(v) => s.serialize_f32(v), - Value::F64(v) => s.serialize_f64(v), - Value::Char(v) => s.serialize_char(v), - Value::String(ref v) => s.serialize_str(v), - // Value::InternedString(v) => s.serialize_u32(v), - Value::Unit => s.serialize_unit(), - Value::Option(None) => s.serialize_none(), - Value::Option(Some(ref v)) => s.serialize_some(v), - Value::Newtype(ref v) => s.serialize_newtype_struct("", v), + Value::Bool(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)), + Value::U8(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)), + Value::U16(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)), + Value::U32(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)), + Value::U64(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)), + Value::I8(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)), + Value::I16(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)), + Value::I32(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)), + Value::I64(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)), + Value::F32(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)), + Value::F64(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)), + Value::Char(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)), + Value::String(ref v) => { + s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)) + } + Value::Bytes(ref v) => { + s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)) + } + Value::Unit => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, ())), + Value::Option(None) => { + s.serialize_newtype_struct("Value", &(self.discriminant() as u8, ())) + } + Value::Option(Some(ref v)) => { + s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)) + } + Value::Newtype(ref v) => { + s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)) + } Value::Seq(ref v) => v.serialize(s), Value::Map(ref v) => v.serialize(s), - Value::Bytes(ref v) => s.serialize_bytes(v), } } }