From d0a58fe6cf6d3b499c2bdf99fb649e60b7e244cc Mon Sep 17 00:00:00 2001 From: Guillaume Binet Date: Thu, 27 Jun 2024 11:49:47 -0500 Subject: [PATCH] datalogger -> unifiedlog --- Cargo.toml | 4 +- copper_helpers/Cargo.toml | 2 +- copper_helpers/src/lib.rs | 4 +- copper_log_reader/Cargo.toml | 2 +- copper_log_reader/src/cli.rs | 6 +-- copper_log_test/Cargo.toml | 11 ++-- copper_log_test/src/main.rs | 30 +++-------- .../Cargo.toml | 2 +- .../src/lib.rs | 54 +++++++++---------- 9 files changed, 47 insertions(+), 68 deletions(-) rename {copper_datalogger => copper_unifiedlog}/Cargo.toml (93%) rename {copper_datalogger => copper_unifiedlog}/src/lib.rs (94%) diff --git a/Cargo.toml b/Cargo.toml index bfba76628..f745ddbb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ "copper_log_test", "copper_log_runtime", "copper_log_reader", - "copper_datalogger", + "copper_unifiedlog", "copper_clock", "copper_traits", "examples/cu_rp_gpio", @@ -17,7 +17,7 @@ members = [ "examples/pluginload", "examples/simplelogger", "examples/v4lsrc" -, "copper_helpers"] + , "copper_helpers"] resolver = "2" diff --git a/copper_helpers/Cargo.toml b/copper_helpers/Cargo.toml index a40f6ec30..f4b35ebb4 100644 --- a/copper_helpers/Cargo.toml +++ b/copper_helpers/Cargo.toml @@ -9,7 +9,7 @@ categories = ["science::robotics"] [dependencies] copper-traits = { path = "../copper_traits" } -copper-datalogger = { path = "../copper_datalogger" } +copper-unifiedlog = { path = "../copper_unifiedlog" } copper-log = { path = "../copper_log" } copper-log-runtime = { path = "../copper_log_runtime" } copper-clock = { path = "../copper_clock" } diff --git a/copper_helpers/src/lib.rs b/copper_helpers/src/lib.rs index 0fa10eee2..34513b789 100644 --- a/copper_helpers/src/lib.rs +++ b/copper_helpers/src/lib.rs @@ -1,8 +1,8 @@ use copper_clock::RobotClock; -use copper_datalogger::{stream_write, DataLogger, DataLoggerBuilder}; use copper_log::default_log_index_dir; use copper_log_runtime::{ExtraTextLogger, LoggerRuntime}; use copper_traits::{CuResult, DataLogType}; +use copper_unifiedlog::{stream_write, UnifiedLogger, UnifiedLoggerBuilder}; use simplelog::{ColorChoice, Config, LevelFilter, TermLogger, TerminalMode}; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -18,7 +18,7 @@ pub fn basic_logger_runtime_setup( datalogger_output_path: &Path, text_log: bool, ) -> CuResult { - let DataLogger::Write(logger) = DataLoggerBuilder::new() + let UnifiedLogger::Write(logger) = UnifiedLoggerBuilder::new() .write(true) .create(true) .file_path(datalogger_output_path) diff --git a/copper_log_reader/Cargo.toml b/copper_log_reader/Cargo.toml index b913c97f5..056b52f46 100644 --- a/copper_log_reader/Cargo.toml +++ b/copper_log_reader/Cargo.toml @@ -15,7 +15,7 @@ path = "src/cli.rs" copper-traits = { path = "../copper_traits" } copper-log = { path = "../copper_log" } copper-clock = { path = "../copper_clock" } -copper-datalogger = { path = "../copper_datalogger" } +copper-unifiedlog = { path = "../copper_unifiedlog" } clap = { version = "4.5.7", features = ["derive"] } rkv = { version = "0.19.0", features = ["lmdb"] } byteorder = "1.5.0" diff --git a/copper_log_reader/src/cli.rs b/copper_log_reader/src/cli.rs index c1a79840f..4913dcff5 100644 --- a/copper_log_reader/src/cli.rs +++ b/copper_log_reader/src/cli.rs @@ -1,7 +1,7 @@ use clap::{Parser, Subcommand}; -use copper_datalogger::{DataLogger, DataLoggerBuilder, DataLoggerIOReader}; use copper_log_reader::full_log_dump; use copper_traits::DataLogType; +use copper_unifiedlog::{UnifiedLogger, UnifiedLoggerBuilder, UnifiedLoggerIOReader}; use std::io::Read; use std::path::PathBuf; @@ -28,7 +28,7 @@ fn main() { PathBuf::from("test/test.copper") }; - let DataLogger::Read(dl) = DataLoggerBuilder::new() + let UnifiedLogger::Read(dl) = UnifiedLoggerBuilder::new() .file_path(&datalog) .build() .expect("Failed to create logger") @@ -36,6 +36,6 @@ fn main() { panic!("Failed to create logger"); }; - let reader = DataLoggerIOReader::new(dl, DataLogType::StructuredLogLine); + let reader = UnifiedLoggerIOReader::new(dl, DataLogType::StructuredLogLine); full_log_dump(reader, &index).expect("Failed to dump log"); } diff --git a/copper_log_test/Cargo.toml b/copper_log_test/Cargo.toml index a75d10e9b..6df4e0ac5 100644 --- a/copper_log_test/Cargo.toml +++ b/copper_log_test/Cargo.toml @@ -8,13 +8,8 @@ categories = ["science::robotics"] [dependencies] copper = { path = "../copper" } -copper-log-derive = { path = "../copper_log_derive" } copper-log = { path = "../copper_log" } -copper-log-runtime = { path = "../copper_log_runtime" } -copper-datalogger = { path = "../copper_datalogger" } -copper-value = { path = "../copper_value" } -serde = { version = "1.0.202", features = ["derive"] } - -[build-dependencies] copper-log-derive = { path = "../copper_log_derive" } - +copper-log-runtime = { path = "../copper_log_runtime" } +copper-helpers = { path = "../copper_helpers" } +serde = { version = "1.0.202", features = ["derive"] } \ No newline at end of file diff --git a/copper_log_test/src/main.rs b/copper_log_test/src/main.rs index 2cefedd1f..a5f57cf63 100644 --- a/copper_log_test/src/main.rs +++ b/copper_log_test/src/main.rs @@ -1,28 +1,14 @@ -use copper::clock::RobotClock; -use copper::DataLogType; -use copper_datalogger::{stream_write, DataLogger, DataLoggerBuilder}; +use copper::monitoring::ScopedAllocCounter; +use copper_helpers::basic_logger_runtime_setup; use copper_log_derive::debug; -use copper_log_runtime::LoggerRuntime; use serde::Serialize; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; fn main() { - let path: PathBuf = PathBuf::from("/tmp/teststructlog.copper"); - let DataLogger::Write(logger) = DataLoggerBuilder::new() - .write(true) - .create(true) - .file_path(&path) - .preallocated_size(100000) - .build() - .expect("Failed to create logger") - else { - panic!("Failed to create logger") - }; - let data_logger = Arc::new(Mutex::new(logger)); - let clock = RobotClock::new(); - let stream = stream_write(data_logger.clone(), DataLogType::StructuredLogLine, 1024); - let mut rt = LoggerRuntime::init(clock, stream, None); + let rt = basic_logger_runtime_setup(&PathBuf::from("/tmp/teststructlog.copper"), true) + .expect("Failed to setup logger."); + debug!("Logger created."); + #[derive(Serialize)] struct Test { a: i32, @@ -30,7 +16,7 @@ fn main() { } let mytuple = (1, "toto", 3.34f64, true, 'a'); { - let hop = copper::monitoring::ScopedAllocCounter::new(); + let hop = ScopedAllocCounter::new(); let gigantic_vec = vec![0u8; 1_000_000]; debug!("Just a string {}", "zarma"); debug!("anonymous param constants {} {}", 42u16, 43u8); @@ -39,6 +25,4 @@ fn main() { debug!("complex tuple", mytuple); debug!("Struct", Test { a: 3, b: 4 }); } - debug!(" AFTER CLOSE {} ", "AFTER CLOSE"); - rt.close(); } diff --git a/copper_datalogger/Cargo.toml b/copper_unifiedlog/Cargo.toml similarity index 93% rename from copper_datalogger/Cargo.toml rename to copper_unifiedlog/Cargo.toml index 36a8e64e2..cd5ac3229 100644 --- a/copper_datalogger/Cargo.toml +++ b/copper_unifiedlog/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "copper-datalogger" +name = "copper-unifiedlog" version = "0.1.0" edition = "2021" license = "Apache-2.0" diff --git a/copper_datalogger/src/lib.rs b/copper_unifiedlog/src/lib.rs similarity index 94% rename from copper_datalogger/src/lib.rs rename to copper_unifiedlog/src/lib.rs index ad0a596b3..71b1b54a6 100644 --- a/copper_datalogger/src/lib.rs +++ b/copper_unifiedlog/src/lib.rs @@ -43,7 +43,7 @@ struct SectionHeader { /// A wrapper around a memory mapped file to write to. struct MmapStream { entry_type: DataLogType, - parent_logger: Arc>, + parent_logger: Arc>, current_slice: &'static mut [u8], current_position: usize, minimum_allocation_amount: usize, @@ -52,7 +52,7 @@ struct MmapStream { impl MmapStream { fn new( entry_type: DataLogType, - parent_logger: Arc>, + parent_logger: Arc>, current_slice: &'static mut [u8], minimum_allocation_amount: usize, ) -> Self { @@ -109,9 +109,9 @@ impl Drop for MmapStream { } } -/// Create a new stream to write to the datalogger. +/// Create a new stream to write to the unifiedlogger. pub fn stream_write( - logger: Arc>, + logger: Arc>, entry_type: DataLogType, minimum_allocation_amount: usize, ) -> impl WriteStream { @@ -130,20 +130,20 @@ pub fn stream_write( const DEFAULT_LOGGER_SIZE: usize = 1024 * 1024 * 1024; // 1GB /// Holder of the read or write side of the datalogger. -pub enum DataLogger { - Read(DataLoggerRead), - Write(DataLoggerWrite), +pub enum UnifiedLogger { + Read(UnifiedLoggerRead), + Write(UnifiedLoggerWrite), } /// Use this builder to create a new DataLogger. -pub struct DataLoggerBuilder { +pub struct UnifiedLoggerBuilder { file_path: Option, preallocated_size: Option, write: bool, create: bool, } -impl DataLoggerBuilder { +impl UnifiedLoggerBuilder { pub fn new() -> Self { Self { file_path: None, @@ -173,7 +173,7 @@ impl DataLoggerBuilder { self } - pub fn build(self) -> io::Result { + pub fn build(self) -> io::Result { if self.create && !self.write { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -213,7 +213,7 @@ impl DataLoggerBuilder { let nb_bytes = encode_into_slice(&main_header, &mut mmap[..], standard()) .expect("Failed to encode main header"); assert!(nb_bytes < page_size); - Ok(DataLogger::Write(DataLoggerWrite { + Ok(UnifiedLogger::Write(UnifiedLoggerWrite { file, mmap_buffer: mmap, page_size, @@ -233,7 +233,7 @@ impl DataLoggerBuilder { "Invalid magic number in main header", )); } - Ok(DataLogger::Read(DataLoggerRead { + Ok(UnifiedLogger::Read(UnifiedLoggerRead { file, mmap_buffer: mmap, reading_position: main_header.first_section_offset as usize, @@ -243,14 +243,14 @@ impl DataLoggerBuilder { } /// A read side of the datalogger. -pub struct DataLoggerRead { +pub struct UnifiedLoggerRead { file: File, mmap_buffer: Mmap, reading_position: usize, } /// A write side of the datalogger. -pub struct DataLoggerWrite { +pub struct UnifiedLoggerWrite { file: File, mmap_buffer: MmapMut, page_size: usize, @@ -259,7 +259,7 @@ pub struct DataLoggerWrite { flushed_until: usize, } -impl DataLoggerWrite { +impl UnifiedLoggerWrite { fn unsure_size(&mut self, size: usize) -> io::Result<()> { // Here it is important that the memory map resizes in place. // According to the documentation this is something unique to Linux to be able to do that. @@ -346,7 +346,7 @@ impl DataLoggerWrite { } } -impl Drop for DataLoggerWrite { +impl Drop for UnifiedLoggerWrite { fn drop(&mut self) { self.add_section(DataLogType::LastEntry, 0); self.flush(); @@ -356,7 +356,7 @@ impl Drop for DataLoggerWrite { } } -impl DataLoggerRead { +impl UnifiedLoggerRead { pub fn read_next_section_type( &mut self, datalogtype: DataLogType, @@ -433,16 +433,16 @@ impl DataLoggerRead { } } -/// This a a convience wrapper around the DataLoggerRead to implement the Read trait. -pub struct DataLoggerIOReader { - logger: DataLoggerRead, +/// This a a convience wrapper around the UnifiedLoggerRead to implement the Read trait. +pub struct UnifiedLoggerIOReader { + logger: UnifiedLoggerRead, log_type: DataLogType, buffer: Vec, buffer_pos: usize, } -impl DataLoggerIOReader { - pub fn new(logger: DataLoggerRead, log_type: DataLogType) -> Self { +impl UnifiedLoggerIOReader { + pub fn new(logger: UnifiedLoggerRead, log_type: DataLogType) -> Self { Self { logger, log_type, @@ -464,7 +464,7 @@ impl DataLoggerIOReader { } } -impl Read for DataLoggerIOReader { +impl Read for UnifiedLoggerIOReader { fn read(&mut self, buf: &mut [u8]) -> io::Result { if self.buffer_pos >= self.buffer.len() { self.fill_buffer()?; @@ -489,9 +489,9 @@ mod tests { use super::*; use std::path::PathBuf; use tempfile::TempDir; - fn make_a_logger(tmp_dir: &TempDir) -> (Arc>, PathBuf) { + fn make_a_logger(tmp_dir: &TempDir) -> (Arc>, PathBuf) { let file_path = tmp_dir.path().join("test.bin"); - let DataLogger::Write(data_logger) = DataLoggerBuilder::new() + let UnifiedLogger::Write(data_logger) = UnifiedLoggerBuilder::new() .write(true) .create(true) .file_path(&file_path) @@ -510,7 +510,7 @@ mod tests { let tmp_dir = TempDir::new().expect("could not create a tmp dir"); let file_path = tmp_dir.path().join("test.bin"); let _used = { - let DataLogger::Write(mut logger) = DataLoggerBuilder::new() + let UnifiedLogger::Write(mut logger) = UnifiedLoggerBuilder::new() .write(true) .create(true) .file_path(&file_path) @@ -598,7 +598,7 @@ mod tests { stream.log(&3u32).unwrap(); } drop(logger); - let DataLogger::Read(mut dl) = DataLoggerBuilder::new() + let UnifiedLogger::Read(mut dl) = UnifiedLoggerBuilder::new() .file_path(&f.to_path_buf()) .build() .expect("Failed to build logger")