mmap, reduced utf8 parsing

Cargo.lock

@@ -12,10 +12,18 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
 name = "brc"
 version = "0.1.0"
 dependencies = [
+ "fxhash",
+ "memmap2",
  "rand",
  "rand_distr",
 ]
 
+[[package]]
+name = "byteorder"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.4"
@@ -42,6 +50,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "fxhash"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
+dependencies = [
+ "byteorder",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.4.3"
@@ -66,6 +83,15 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 
+[[package]]
+name = "memmap2"
+version = "0.9.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"

Cargo.toml

@@ -7,6 +7,8 @@ default-run = "brc"
 [dependencies]
 rand_distr = "*"
 rand = "*"
+fxhash = "*"
+memmap2 = "*"
 
 [[bin]]
 name = "generate"

benchmark.txt

@@ -2,3 +2,5 @@
 110 lto, codegen, abort
 107 nightly, build-std
 109 native cpu (doesn't seem to work)
+ 95 FxHashMap
+ 75 mmap, reduced UTF8 parsing (names)

src/main.rs

@@ -1,17 +1,40 @@
-use std::{collections::HashMap, io::BufRead};
+use fxhash::FxHashMap;
+use memmap2::MmapOptions;
 
 fn main() {
-    let file = std::io::BufReader::new(std::fs::File::open("measurements.txt").unwrap());
+    let mut data = FxHashMap::default();
 
-    let mut data = HashMap::new();
+    let file = std::fs::File::open("measurements.txt").unwrap();
+    // SAFETY: we are not going to modify the file while the program is running. No lock required
+    // for now.
+    let mmap = unsafe { MmapOptions::new().map(&file) }.unwrap();
+    mmap.advise(memmap2::Advice::Sequential).unwrap();
 
-    for result in file.lines() {
+    let mut pointer = 0;
-        let line = result.unwrap();
+    let mut length = 0;
 
-        let (station, value) = line.split_once(';').unwrap();
+    loop {
-        let station = station.to_string();
+        length = 0;
+        while let Some(ch) = &mmap.get(pointer + length) {
+            if **ch == b'\n' {
+                break;
+            } else {
+                length += 1;
+            }
+        }
+
+        if length == 0 || pointer + length == mmap.len() {
+            break;
+        }
+        let line = &mmap[pointer..pointer + length];
+
+        let mut split = line.split(|&c| c == b';');
+        let station = split.next().unwrap();
+        let parsed: f64 = String::from_utf8(split.next().unwrap().to_vec())
+            .unwrap()
+            .parse()
+            .unwrap();
 
-        let parsed: f64 = value.parse().unwrap();
         data.entry(station)
             .and_modify(|values: &mut (f64, f64, usize, f64)| {
                 values.0 = values.0.min(parsed);
@@ -20,6 +43,8 @@ fn main() {
                 values.3 = values.3.max(parsed);
             })
             .or_insert((parsed, parsed, 1_usize, parsed));
+
+        pointer += length + 1;
     }
 
     let mut stations: Vec<_> = data.keys().collect();
@@ -30,6 +55,9 @@ fn main() {
     let mut iter = stations.into_iter().peekable();
     while let Some(s) = iter.next() {
         let entry = data.get(s).unwrap();
+
+        let s = String::from_utf8(s.to_vec()).unwrap();
+
         print!(
             "{s}={:.1}/{:.1}/{:.1}",
             entry.0,