Logger serialization fixes.

There are still an issue with Value and deserializing them with serde but serialization looks ok.
copper-project · May 30, 2024 · 0595669 · 0595669
1 parent 0e32e4c
commit 0595669
Show file tree

Hide file tree

Showing 14 changed files with 113 additions and 56 deletions.
diff --git a/copper/src/config.rs b/copper/src/config.rs
@@ -337,7 +337,6 @@ pub fn read_configuration(config_filename: &str) -> CuResult<CuConfig> {
 // tests
 #[cfg(test)]
 mod tests {
-    use uom::si::power::watt;
     use uom::si::time::millisecond;
     use uom::si::time::second;
 

diff --git a/copper_log/src/lib.rs b/copper_log/src/lib.rs
@@ -5,7 +5,6 @@ extern crate proc_macro;
 use crate::index::{check_and_insert, intern_string};
 use proc_macro::TokenStream;
 use quote::quote;
-use std::fmt::Display;
 use syn::parse::Parser;
 use syn::Token;
 use syn::{Expr, ExprAssign, ExprLit, Lit};

diff --git a/copper_log_reader/Cargo.toml b/copper_log_reader/Cargo.toml
@@ -7,4 +7,5 @@ edition = "2021"
 copper-log-runtime = { path = "../copper_log_runtime" }
 rkv = { version = "0.19.0", features = ["lmdb"] }
 byteorder = "1.5.0"
-bincode = "2.0.0-rc.3"
+bincode = { version = "2.0.0-rc.3", features = ["serde", "derive"] }
+strfmt = "0.2.0"
diff --git a/copper_log_reader/src/lib.rs b/copper_log_reader/src/lib.rs
@@ -4,17 +4,19 @@ use byteorder::{ByteOrder, LittleEndian};
 use copper_log_runtime::CuLogEntry;
 use rkv::backend::Lmdb;
 use rkv::{Rkv, StoreOptions, Writer};
+use std::collections::HashMap;
 use std::io::Read;
 use std::path::Path;
+use strfmt::strfmt;
 
-pub fn extract_copper_log_index(mut src: impl Read, index: &Path) {
+pub fn extract_copper_log_with_index(mut src: impl Read, index: &Path) {
     let mut all_strings = Vec::<String>::new();
     let env = Rkv::new::<Lmdb>(&index).unwrap();
     let index_to_string = env
         .open_single("index_to_string", StoreOptions::default())
         .expect("Failed to open index_to_string store");
-    let db_eader = env.read().unwrap();
-    let ri = index_to_string.iter_start(&db_eader);
+    let db_reader = env.read().unwrap();
+    let ri = index_to_string.iter_start(&db_reader);
     let mut i = ri.expect("Failed to start iterator");
     while let Some(Ok(v)) = i.next() {
         let (k, v) = v;
@@ -29,42 +31,48 @@ pub fn extract_copper_log_index(mut src: impl Read, index: &Path) {
             println!("{} -> {}", index, s);
         }
     }
-    let entry = decode_from_std_read::<CuLogEntry, _, _>(&mut src, standard());
-    let entry = entry.expect("Failed to decode CuLogEntry");
-    println!(
-        "Entry: {} -> {} with params {:?}",
-        entry.msg_index, all_strings[entry.msg_index as usize], entry.params
-    );
-    //FIXME: Entry: 1 -> Just a string {} with params [I16(61)]
-    compile_error!("ici");
+    loop {
+        let entry = decode_from_std_read::<CuLogEntry, _, _>(&mut src, standard());
+        if entry.is_err() {
+            break;
+        }
+        let entry = entry.unwrap();
+        let mut format_string = all_strings[entry.msg_index as usize].clone();
+        let mut vars = HashMap::new();
+
+        for (i, param) in entry.params.iter().enumerate() {
+            let param_as_string = format!("{}", param);
+            if entry.paramname_indexes[i] == 0 {
+                // Anonymous parameter
+                format_string = format_string.replacen("{}", &param_as_string, 1);
+            } else {
+                // Named parameter
+                let name = all_strings[entry.paramname_indexes[i] as usize].clone();
+                vars.insert(name, param_as_string);
+            }
+        }
+
+        // Use strfmt to replace named parameters
+        let result = strfmt(&format_string, &vars).unwrap();
+        println!("Copper: {}", result);
+    }
 }
 
 #[cfg(test)]
 mod tests {
 
     use super::*;
     use std::io::Cursor;
-    /*
-        Just a string {} CuLogEntry { msg_index: 1, paramname_indexes: [0], params: [String("zarma")] }
-        anonymous param constants {} {} CuLogEntry { msg_index: 2, paramname_indexes: [0, 0], params: [U16(42), U8(43)] }
-        named param constants {} {} CuLogEntry { msg_index: 3, paramname_indexes: [4, 5], params: [I32(3), I32(2)] }
-        mixed named param constants, {} {} {} CuLogEntry { msg_index: 6, paramname_indexes: [0, 4, 5], params: [I32(54), I32(3), I32(2)] }
-        complex tuple CuLogEntry { msg_index: 7, paramname_indexes: [0], params: [Seq([I32(1), String("toto"), F64(3.34), Bool(true), Char('a')])] }
-        Struct CuLogEntry { msg_index: 8, paramname_indexes: [0], params: [Map({String("a"): I32(3), String("b"): I32(4)})] }
-        Allocations: +{}B -{}B CuLogEntry { msg_index: 1, paramname_indexes: [2, 3], params: [U64(1003932), U64(1002876)] }
-         AFTER CLOSE {}  CuLogEntry { msg_index: 10, paramname_indexes: [0], params: [String("AFTER CLOSE")] }
-    */
-
-    // const stored_log: &[u8] = include_bytes!("../test/teststructlog.copper");
 
     #[test]
     fn test_extract_copper_log() {
-        let hex_string = "01 01 00 01 05 7A 61 72 6D 61 02 02 00 00 02 2A 2B 03 02 04 05 02 06 04 06 03 00 04 05 03 6C 06 04 07 01 00 01 05 02 04 74 6F 74 6F B8 1E 85 EB 51 B8 0A 40 01 61 08 01 00 01 02 01 61 06 01 62 08 01 02 02 03 02 FC 9C 51 0F 00 FC 7C 4D 0F 00 0A 01 00 01 0B 41 46 54 45 52 20 43 4C 4F 53 45 43 4C 4F 53 45 00 00 00 00";
+        let hex_string = "01 01 00 01 0C 05 7A 61 72 6D 61";
         let bytes: Vec<u8> = hex_string
             .split_whitespace()
             .map(|s| u8::from_str_radix(s, 16).expect("Parse error"))
             .collect();
-        let mut reader = Cursor::new(bytes.as_slice());
-        extract_copper_log_index(reader, Path::new("test/copper_log_index"));
+
+        let reader = Cursor::new(bytes.as_slice());
+        extract_copper_log_with_index(reader, Path::new("test/copper_log_index"));
     }
 }
diff --git a/copper_log_reader/test/copper_log_index/data.mdb b/copper_log_reader/test/copper_log_index/data.mdb
diff --git a/copper_log_reader/test/copper_log_index/lock.mdb b/copper_log_reader/test/copper_log_index/lock.mdb
diff --git a/copper_log_reader/test/teststructlog.copper b/copper_log_reader/test/teststructlog.copper
diff --git a/copper_log_runtime/src/lib.rs b/copper_log_runtime/src/lib.rs
@@ -1,10 +1,10 @@
-use bincode::enc::write::Writer;
 use bincode_derive::{Decode, Encode};
 use copper_traits::{CuResult, Stream};
 pub use copper_value as value;
 use copper_value::Value;
 use kanal::{bounded, Sender};
 use once_cell::sync::OnceCell;
+use serde::{Deserialize, Serialize};
 use std::fmt::Display;
 use std::io::{stdout, Write};
 use std::sync::{Arc, Mutex};
@@ -19,7 +19,7 @@ pub const ANONYMOUS: u32 = 0;
 /// The lifetime of this struct is the lifetime of the logger.
 pub struct LoggerRuntime {}
 
-#[derive(Debug, Encode, Decode)]
+#[derive(Debug, Encode, Decode, Serialize, Deserialize, PartialEq)]
 pub struct CuLogEntry {
     pub msg_index: u32,
     pub paramname_indexes: Vec<u32>,
@@ -74,8 +74,6 @@ impl Drop for LoggerRuntime {
 
 fn initialize_queue(mut destination: impl Stream + 'static) -> Sender<CuLogEntry> {
     let (sender, receiver) = bounded::<CuLogEntry>(100);
-    let config = bincode::config::standard();
-
     let handle = thread::spawn(move || loop {
         if let Ok(data) = receiver.recv() {
             if let Err(err) = destination.log(&data) {
@@ -110,3 +108,42 @@ pub fn log(entry: CuLogEntry) -> CuResult<()> {
         Err("Logger not initialized.".into())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::CuLogEntry;
+    use bincode::config::standard;
+    use copper_value::Value;
+
+    #[test]
+    fn test_encode_decode_structured_log() {
+        let log_entry = CuLogEntry {
+            msg_index: 1,
+            paramname_indexes: vec![2, 3],
+            params: vec![Value::String("test".to_string())],
+        };
+        let encoded = bincode::encode_to_vec(&log_entry, standard()).unwrap();
+        println!("{:?}", encoded);
+        let decoded_tuple: (CuLogEntry, usize) =
+            bincode::decode_from_slice(&encoded, standard()).unwrap();
+        assert_eq!(log_entry, decoded_tuple.0);
+    }
+
+    #[test]
+    #[ignore] // FIXME: the serde sere works but deser doesn't with the current implementation
+    fn test_serialize_deserialize_structured_log() {
+        // tests compatibility with std serde
+        let log_entry = CuLogEntry {
+            msg_index: 1,
+            paramname_indexes: vec![2, 3],
+            params: vec![Value::String("test".to_string())],
+        };
+        let v = bincode::serde::encode_to_vec(&log_entry, standard()).unwrap();
+        // Should be [1, 2, 2, 3, 1, 12, 4, 116, 101, 115, 116]
+        println!("{:?}", v);
+        let decoded: (CuLogEntry, usize) =
+            bincode::serde::decode_from_slice(v.as_slice(), standard()).unwrap();
+        assert_eq!(log_entry, decoded.0);
+    }
+}
diff --git a/copper_value/Cargo.toml b/copper_value/Cargo.toml
@@ -12,7 +12,7 @@ license = "MIT"
 [dependencies]
 serde = "1.0.202"
 ordered-float = "4.2.0"
-bincode = "2.0.0-rc.3"
+bincode = { version = "2.0.0-rc.3", features = ["serde", "derive"] }
 
 [dev-dependencies]
-serde_derive = "1.0.202"
+serde_derive = "1.0.203"
diff --git a/copper_value/src/bdec.rs b/copper_value/src/bdec.rs
@@ -29,7 +29,7 @@ impl Decode for Value {
             16 => Ok(Value::Map(BTreeMap::<Value, Value>::decode(decoder)?)),
             17 => Ok(Value::Option(Option::<Box<Value>>::decode(decoder)?)),
             18 => Ok(Value::Newtype(Box::<Value>::decode(decoder)?)),
-            _ => Err(DecodeError::Other("Unknown Value variant")),
+            _ => Err(DecodeError::Other("Unknown Value varian")),
         }
     }
 }

diff --git a/copper_value/src/benc.rs b/copper_value/src/benc.rs
@@ -6,6 +6,8 @@ use bincode::error::EncodeError;
 
 impl Encode for Value {
     fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
+        let discriminant = self.discriminant() as u8;
+        discriminant.encode(encoder)?;
         match self {
             Value::U8(v) => v.to_owned().encode(encoder),
             Value::U16(v) => v.to_owned().encode(encoder),

diff --git a/copper_value/src/de.rs b/copper_value/src/de.rs
@@ -283,6 +283,7 @@ impl<'de> de::Visitor<'de> for ValueVisitor {
     }
 
     fn visit_newtype_struct<D: de::Deserializer<'de>>(self, d: D) -> Result<Value, D::Error> {
+        println!("visit_newtype_struct");
         d.deserialize_any(ValueVisitor)
             .map(|v| Value::Newtype(Box::new(v)))
     }
@@ -314,6 +315,7 @@ impl<'de> de::Visitor<'de> for ValueVisitor {
 
 impl<'de> de::Deserialize<'de> for Value {
     fn deserialize<D: de::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
+        println!("Value::deserialize");
         d.deserialize_any(ValueVisitor)
     }
 }

diff --git a/copper_value/src/lib.rs b/copper_value/src/lib.rs
@@ -36,14 +36,14 @@ pub enum Value {
 
     Char(char),
     String(String),
-    // InternedString(u32),
     Unit,
     Option(Option<Box<Value>>),
     Newtype(Box<Value>),
     Seq(Vec<Value>),
     Map(BTreeMap<Value, Value>),
     Bytes(Vec<u8>),
 }
+
 impl Display for Value {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {

diff --git a/copper_value/src/ser.rs b/copper_value/src/ser.rs
@@ -33,27 +33,36 @@ impl ser::Error for SerializerError {
 impl ser::Serialize for Value {
     fn serialize<S: ser::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
         match *self {
-            Value::Bool(v) => s.serialize_bool(v),
-            Value::U8(v) => s.serialize_u8(v),
-            Value::U16(v) => s.serialize_u16(v),
-            Value::U32(v) => s.serialize_u32(v),
-            Value::U64(v) => s.serialize_u64(v),
-            Value::I8(v) => s.serialize_i8(v),
-            Value::I16(v) => s.serialize_i16(v),
-            Value::I32(v) => s.serialize_i32(v),
-            Value::I64(v) => s.serialize_i64(v),
-            Value::F32(v) => s.serialize_f32(v),
-            Value::F64(v) => s.serialize_f64(v),
-            Value::Char(v) => s.serialize_char(v),
-            Value::String(ref v) => s.serialize_str(v),
-            // Value::InternedString(v) => s.serialize_u32(v),
-            Value::Unit => s.serialize_unit(),
-            Value::Option(None) => s.serialize_none(),
-            Value::Option(Some(ref v)) => s.serialize_some(v),
-            Value::Newtype(ref v) => s.serialize_newtype_struct("", v),
+            Value::Bool(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)),
+            Value::U8(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)),
+            Value::U16(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)),
+            Value::U32(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)),
+            Value::U64(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)),
+            Value::I8(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)),
+            Value::I16(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)),
+            Value::I32(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)),
+            Value::I64(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)),
+            Value::F32(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)),
+            Value::F64(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)),
+            Value::Char(v) => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v)),
+            Value::String(ref v) => {
+                s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v))
+            }
+            Value::Bytes(ref v) => {
+                s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v))
+            }
+            Value::Unit => s.serialize_newtype_struct("Value", &(self.discriminant() as u8, ())),
+            Value::Option(None) => {
+                s.serialize_newtype_struct("Value", &(self.discriminant() as u8, ()))
+            }
+            Value::Option(Some(ref v)) => {
+                s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v))
+            }
+            Value::Newtype(ref v) => {
+                s.serialize_newtype_struct("Value", &(self.discriminant() as u8, v))
+            }
             Value::Seq(ref v) => v.serialize(s),
             Value::Map(ref v) => v.serialize(s),
-            Value::Bytes(ref v) => s.serialize_bytes(v),
         }
     }
 }