diff --git a/Cargo.lock b/Cargo.lock index 802bf95..d3f0514 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,10 +12,18 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" name = "brc" version = "0.1.0" dependencies = [ + "fxhash", + "memmap2", "rand", "rand_distr", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "cfg-if" version = "1.0.4" @@ -42,6 +50,15 @@ dependencies = [ "libc", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -66,6 +83,15 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "num-traits" version = "0.2.19" diff --git a/Cargo.toml b/Cargo.toml index d649057..44fccb3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,8 @@ default-run = "brc" [dependencies] rand_distr = "*" rand = "*" +fxhash = "*" +memmap2 = "*" [[bin]] name = "generate" diff --git a/benchmark.txt b/benchmark.txt index 58ef695..857cfd7 100644 --- a/benchmark.txt +++ b/benchmark.txt @@ -2,3 +2,5 @@ 110 lto, codegen, abort 107 nightly, build-std 109 native cpu (doesn't seem to work) + 95 FxHashMap + 75 mmap, reduced UTF8 parsing (names) diff --git a/src/main.rs b/src/main.rs index aee6529..ce7fed0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,17 +1,40 @@ -use std::{collections::HashMap, io::BufRead}; +use fxhash::FxHashMap; +use memmap2::MmapOptions; fn main() { - let file = std::io::BufReader::new(std::fs::File::open("measurements.txt").unwrap()); + let mut data = FxHashMap::default(); - let mut data = HashMap::new(); + let file = std::fs::File::open("measurements.txt").unwrap(); + // SAFETY: we are not going to modify the file while the program is running. No lock required + // for now. + let mmap = unsafe { MmapOptions::new().map(&file) }.unwrap(); + mmap.advise(memmap2::Advice::Sequential).unwrap(); - for result in file.lines() { - let line = result.unwrap(); + let mut pointer = 0; + let mut length = 0; - let (station, value) = line.split_once(';').unwrap(); - let station = station.to_string(); + loop { + length = 0; + while let Some(ch) = &mmap.get(pointer + length) { + if **ch == b'\n' { + break; + } else { + length += 1; + } + } + + if length == 0 || pointer + length == mmap.len() { + break; + } + let line = &mmap[pointer..pointer + length]; + + let mut split = line.split(|&c| c == b';'); + let station = split.next().unwrap(); + let parsed: f64 = String::from_utf8(split.next().unwrap().to_vec()) + .unwrap() + .parse() + .unwrap(); - let parsed: f64 = value.parse().unwrap(); data.entry(station) .and_modify(|values: &mut (f64, f64, usize, f64)| { values.0 = values.0.min(parsed); @@ -20,6 +43,8 @@ fn main() { values.3 = values.3.max(parsed); }) .or_insert((parsed, parsed, 1_usize, parsed)); + + pointer += length + 1; } let mut stations: Vec<_> = data.keys().collect(); @@ -30,6 +55,9 @@ fn main() { let mut iter = stations.into_iter().peekable(); while let Some(s) = iter.next() { let entry = data.get(s).unwrap(); + + let s = String::from_utf8(s.to_vec()).unwrap(); + print!( "{s}={:.1}/{:.1}/{:.1}", entry.0,