wip

Cargo.toml

@@ -23,7 +23,7 @@ high_precision = []
 
 [dependencies]
 byteorder = "1.5.0"
-fjall = "2.3.2"
+fjall = "2.4.0"
 log = "0.4.22"
 logos = "0.14.0"
 # nom = "7.1.3"

README.md

@@ -45,15 +45,22 @@ Data points are f32s by default, but can be switched to f64 using the `high_prec
 
 ## Benchmark: 1 billion data points
 
-Hyper mode, jemalloc, i9 11900k:
+Hyper mode, jemalloc, i9 11900k, Samsung PM9A3:
 
 ```
-ingested 1 billion in 765s
-write speed: 1307189 writes per second
-peak mem: 167 MiB
+ingested 1 billion in 769s
+write speed: 1300390 writes per second
+peak mem: 158 MiB
 disk space: 10 GiB
-query [1M latest data points] in 176ms
-reopened DB in 128ms
+query [1M latest data points] in 197ms
+reopened DB in 140ms
+```
+
+Run with:
+
+```bash
+cd billion
+cargo run -r
 ```
 
 ## Basic usage
@@ -97,3 +104,5 @@ let buckets = db
 
 println!("{buckets:#?}");
 ```
+
+![Timeseries parameters](./timeseries.svg)

billion/src/main.rs

@@ -29,7 +29,9 @@ fn main() -> talna::Result<()> {
 
     {
         let db = Database::builder()
-            .cache_size_mib(8) // TODO: cache currently bloats memory a lot because of Arcs in block cache
+            // TODO: cache currently bloats memory a lot because of Arcs in block cache
+            // TODO: timeseries would probably benefit a lot from a CompressedBlockCache (to skip I/O, but not CPU work)
+            .cache_size_mib(8)
             .hyper_mode(true)
             .open(path)?;
 
@@ -55,7 +57,7 @@ fn main() -> talna::Result<()> {
                 let proc = sys.processes();
                 let child = proc.get(&pid).unwrap();
                 max_memory_bytes.fetch_max(child.memory(), std::sync::atomic::Ordering::Relaxed);
-                std::thread::sleep(Duration::from_millis(100));
+                std::thread::sleep(Duration::from_millis(1_000));
             }
         });
 

src/db.rs

@@ -63,10 +63,16 @@ impl Database {
     }
 
     pub(crate) fn from_keyspace(keyspace: TxKeyspace, hyper_mode: bool) -> crate::Result<Self> {
+        log::info!("Opening database using existing keyspace");
+
+        log::info!("Opening meta partitions");
+
         let tag_index = TagIndex::new(&keyspace)?;
         let tag_sets = TagSets::new(&keyspace)?;
         let series_mapping = SeriesMapping::new(&keyspace)?;
 
+        log::info!("Opening data partition");
+
         let data = keyspace
             .open_partition(
                 "_talna#data",