From 0e32e4c9dc92e00d6f75f5fe4e041f4d8239807e Mon Sep 17 00:00:00 2001 From: Guillaume Binet Date: Wed, 29 May 2024 21:08:20 -0400 Subject: [PATCH] WIP struct log reading --- Cargo.toml | 2 +- copper_datalogger/src/lib.rs | 70 ++++++++++-------- copper_log/Cargo.toml | 7 +- copper_log/src/lib.rs | 1 + copper_log/src/log_extract.rs | 3 - copper_log_reader/Cargo.toml | 10 +++ copper_log_reader/src/lib.rs | 70 ++++++++++++++++++ .../test/copper_log_index/data.mdb | Bin 0 -> 73728 bytes .../test/copper_log_index/lock.mdb | Bin 0 -> 8192 bytes copper_log_reader/test/teststructlog.copper | Bin 0 -> 5126 bytes copper_log_runtime/src/lib.rs | 13 ++-- copper_log_test/src/main.rs | 10 +-- copper_traits/src/lib.rs | 8 +- 13 files changed, 145 insertions(+), 49 deletions(-) delete mode 100644 copper_log/src/log_extract.rs create mode 100644 copper_log_reader/Cargo.toml create mode 100644 copper_log_reader/src/lib.rs create mode 100644 copper_log_reader/test/copper_log_index/data.mdb create mode 100644 copper_log_reader/test/copper_log_index/lock.mdb create mode 100644 copper_log_reader/test/teststructlog.copper diff --git a/Cargo.toml b/Cargo.toml index dc4f5ed9b..fdc3933ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,5 +11,5 @@ members = ["copper", "copper_log_runtime", "copper_datalogger", "copper_clock", - "copper_traits"] + "copper_traits", "copper_log_reader"] resolver = "2" diff --git a/copper_datalogger/src/lib.rs b/copper_datalogger/src/lib.rs index 66568a030..607f961e4 100644 --- a/copper_datalogger/src/lib.rs +++ b/copper_datalogger/src/lib.rs @@ -19,7 +19,7 @@ use bincode::{decode_from_reader, decode_from_slice}; use bincode_derive::Decode as dDecode; use bincode_derive::Encode as dEncode; -use copper::{CuError, CuResult, Stream}; +use copper::{CuError, CuResult, DataLogType, Stream}; const MAIN_MAGIC: [u8; 4] = [0xB4, 0xA5, 0x50, 0xFF]; @@ -31,21 +31,15 @@ struct MainHeader { first_section_offset: u16, // This is to align with a page at write time. } -#[derive(dEncode, dDecode, Copy, Clone)] -pub enum EntryType { - StructuredLogLine, - CopperList, -} - #[derive(dEncode, dDecode)] struct SectionHeader { magic: [u8; 2], - entry_type: EntryType, + entry_type: DataLogType, section_size: u32, // offset of section_magic + section_size -> should be the index of the next section_magic } struct MmapStream { - entry_type: EntryType, + entry_type: DataLogType, parent_logger: Arc>, current_slice: &'static mut [u8], current_position: usize, @@ -54,7 +48,7 @@ struct MmapStream { impl MmapStream { fn new( - entry_type: EntryType, + entry_type: DataLogType, parent_logger: Arc>, current_slice: &'static mut [u8], minimum_allocation_amount: usize, @@ -114,7 +108,7 @@ impl Drop for MmapStream { pub fn stream( logger: Arc>, - entry_type: EntryType, + entry_type: DataLogType, minimum_allocation_amount: usize, ) -> impl Stream { let clone = logger.clone(); @@ -208,7 +202,7 @@ impl DataLogger { } /// The returned slice is section_size or greater. - fn add_section(&mut self, entry_type: EntryType, section_size: usize) -> &mut [u8] { + fn add_section(&mut self, entry_type: DataLogType, section_size: usize) -> &mut [u8] { // align current_position to the next page self.current_global_position = (self.current_global_position + self.page_size - 1) & !(self.page_size - 1); @@ -264,24 +258,41 @@ impl Drop for DataLogger { // Section iterator returning the [u8] of the current section pub struct SectionIterator { reader: BufReader, + datalogtype: Option, } impl Iterator for SectionIterator { type Item = Vec; fn next(&mut self) -> Option { - let section_header: SectionHeader = decode_from_reader(&mut self.reader, standard()) - .expect("Failed to decode section header"); - let mut section = vec![0; section_header.section_size as usize]; - self.reader - .read_exact(section.as_mut_slice()) - .expect("Failed to read section"); - Some(section) + let answer: Option> = loop { + if let Ok(section_header) = + decode_from_reader::(&mut self.reader, standard()) + { + let mut section = vec![0; section_header.section_size as usize]; + let read = self.reader.read_exact(section.as_mut_slice()); + if read.is_err() { + break None; + } + if let Some(datalogtype) = self.datalogtype { + if section_header.entry_type == datalogtype { + break Some(section); + } + } + } else { + break None; + } + }; + answer } } // make an iterator to read back a serialzed datalogger from a file -pub fn read_datalogger(file_path: &Path) -> io::Result>> { +// optionally filter by type +pub fn read_datalogger( + file_path: &Path, + datalogtype: Option, +) -> io::Result>> { let mut file = OpenOptions::new().read(true).open(file_path)?; let mut header = [0; 4096]; let s = file.read(&mut header).unwrap(); @@ -306,6 +317,7 @@ pub fn read_datalogger(file_path: &Path) -> io::Result::new(); + let env = Rkv::new::(&index).unwrap(); + let index_to_string = env + .open_single("index_to_string", StoreOptions::default()) + .expect("Failed to open index_to_string store"); + let db_eader = env.read().unwrap(); + let ri = index_to_string.iter_start(&db_eader); + let mut i = ri.expect("Failed to start iterator"); + while let Some(Ok(v)) = i.next() { + let (k, v) = v; + let index = LittleEndian::read_u32(&k) as usize; + + if let rkv::Value::Str(s) = v { + if all_strings.len() <= index as usize { + all_strings.resize(index as usize + 1, String::new()); + } + + all_strings[index] = s.to_string(); + println!("{} -> {}", index, s); + } + } + let entry = decode_from_std_read::(&mut src, standard()); + let entry = entry.expect("Failed to decode CuLogEntry"); + println!( + "Entry: {} -> {} with params {:?}", + entry.msg_index, all_strings[entry.msg_index as usize], entry.params + ); + //FIXME: Entry: 1 -> Just a string {} with params [I16(61)] + compile_error!("ici"); +} + +#[cfg(test)] +mod tests { + + use super::*; + use std::io::Cursor; + /* + Just a string {} CuLogEntry { msg_index: 1, paramname_indexes: [0], params: [String("zarma")] } + anonymous param constants {} {} CuLogEntry { msg_index: 2, paramname_indexes: [0, 0], params: [U16(42), U8(43)] } + named param constants {} {} CuLogEntry { msg_index: 3, paramname_indexes: [4, 5], params: [I32(3), I32(2)] } + mixed named param constants, {} {} {} CuLogEntry { msg_index: 6, paramname_indexes: [0, 4, 5], params: [I32(54), I32(3), I32(2)] } + complex tuple CuLogEntry { msg_index: 7, paramname_indexes: [0], params: [Seq([I32(1), String("toto"), F64(3.34), Bool(true), Char('a')])] } + Struct CuLogEntry { msg_index: 8, paramname_indexes: [0], params: [Map({String("a"): I32(3), String("b"): I32(4)})] } + Allocations: +{}B -{}B CuLogEntry { msg_index: 1, paramname_indexes: [2, 3], params: [U64(1003932), U64(1002876)] } + AFTER CLOSE {} CuLogEntry { msg_index: 10, paramname_indexes: [0], params: [String("AFTER CLOSE")] } + */ + + // const stored_log: &[u8] = include_bytes!("../test/teststructlog.copper"); + + #[test] + fn test_extract_copper_log() { + let hex_string = "01 01 00 01 05 7A 61 72 6D 61 02 02 00 00 02 2A 2B 03 02 04 05 02 06 04 06 03 00 04 05 03 6C 06 04 07 01 00 01 05 02 04 74 6F 74 6F B8 1E 85 EB 51 B8 0A 40 01 61 08 01 00 01 02 01 61 06 01 62 08 01 02 02 03 02 FC 9C 51 0F 00 FC 7C 4D 0F 00 0A 01 00 01 0B 41 46 54 45 52 20 43 4C 4F 53 45 43 4C 4F 53 45 00 00 00 00"; + let bytes: Vec = hex_string + .split_whitespace() + .map(|s| u8::from_str_radix(s, 16).expect("Parse error")) + .collect(); + let mut reader = Cursor::new(bytes.as_slice()); + extract_copper_log_index(reader, Path::new("test/copper_log_index")); + } +} diff --git a/copper_log_reader/test/copper_log_index/data.mdb b/copper_log_reader/test/copper_log_index/data.mdb new file mode 100644 index 0000000000000000000000000000000000000000..5023d02911430d7d13560f973da9935e70a8c98b GIT binary patch literal 73728 zcmeI*KaU$l7{~F+Vsg$U*>O@pAfRlKNLL_qWrS!D3PprOL@20YoI5MBbY7qAHFAV# z@(L*O0;s4EqDB!)Nr@;at~#XziC4hRpPyG*@ZIUov9g!16=l}m+40`TZk~Cb85^TV z+aK&b{rlH5ddat)@T_dwraDs{h10Eg&FirZP1UP9dcOaK*#A;&--#^^0tg_000Iag zfB*srAb#YOo+C%kw%dD6E*4A9_^07S~4j+E7sdMMbmlmCK z`S3;O+;EG|oju^(YX_Wj@%*AOT@Jo}&W#lw#Q&0@MF0T=5I_I{1Q0*~0R#|essQo7 zW;yEdzv}Sa{K$3lIQ`T9V86Cs+1s{b|4RQze@P#wP5solZv+rP009ILKmY**5I_Kd zms{X1e|s_K9+K?FWFLL<#jVe?k3aqF?yc;bZ!>c~_VU?xm9bB6nozRdDtD_Zv)wL@ z?jCoG@pP*&Z}=xI<1_DCdcHLp?(XzwSurh}_hS#Af4||M7&9Eq`rBFF8rFCdGJ=KPdb8yA9(d=8Cyv@7Uk%_jZ^1 z|AzL!jL@A^JA+l?9q?$UIxyXrFu4-z&);W;I+k zK0MT4lLYFofIiGG7~^I^+#~b5(UJu??v-mIY6Q~Ib3z)Xr%y&qUvzf8FN3;@8ao|UZTqT@XFPL(*KE~{?PZD zIVAO;X!aoV@0fbpKT&;xuY?Gn8tea8`|ai35I_I{1Q0*~0R#|0009J+FR;4&e|>L2 zzY0N<2(^6!{VIe7=luGE1?SvsKz$v5s0V2Ix`9RK{7Qh@nDgn6@=EIem;cRUL_8{y(D8AbNrQ;(v+IB7gt_2q1s}0tg_000Ia!SK#FFzv}T7x1W~yU+WTx|C{^m=FSm7009IL zKmY**5I_I{1SD|s_&N=M)?ufB*srAbHESKeQF@?*yTnm*7rg2&ze cPvdAg!lC>Bxi<_Koj2WX+$FHvO|&85zoxqb7ytkO literal 0 HcmV?d00001 diff --git a/copper_log_reader/test/copper_log_index/lock.mdb b/copper_log_reader/test/copper_log_index/lock.mdb new file mode 100644 index 0000000000000000000000000000000000000000..9e12e2a890036bcd21fe9d54b65e1a5e48f54bdf GIT binary patch literal 8192 zcmeIu!3lsc3> = OnceCell::new(); #[allow(dead_code)] pub const ANONYMOUS: u32 = 0; +/// The lifetime of this struct is the lifetime of the logger. +pub struct LoggerRuntime {} + #[derive(Debug, Encode, Decode)] pub struct CuLogEntry { pub msg_index: u32, @@ -47,10 +50,6 @@ impl CuLogEntry { self.params.push(param); } } - -/// The lifetime of this struct is the lifetime of the logger. -pub struct LoggerRuntime {} - impl LoggerRuntime { pub fn init(destination: impl Stream + 'static) -> Self { QUEUE @@ -79,7 +78,9 @@ fn initialize_queue(mut destination: impl Stream + 'static) -> Sender = Result; pub trait Stream: Sync + Send { fn log(&mut self, obj: &impl Encode) -> CuResult<()>; } + +#[derive(dEncode, dDecode, Copy, Clone, Debug, PartialEq)] +pub enum DataLogType { + StructuredLogLine, + CopperList, +}