decoupled dataset creation and init for finer control + some minor cleaning

Cargo.toml

@@ -15,13 +15,13 @@ edition = "2021"
 crate-type = ["cdylib", "lib"]
 
 [target.'cfg(not(target = "x86_64-unknown-linux-musl"))'.build-dependencies]
-bindgen = "0.71"
+bindgen = "0.72"
  
 [target.'cfg(target = "x86_64-unknown-linux-musl")'.build-dependencies]
-bindgen = {version = "0.71", default-features = false, features = ["static"]}
+bindgen = {version = "0.72", default-features = false, features = ["static"]}
 
 [dependencies]
-bitflags = "2.8"
+bitflags = "2.10"
 libc = "0.2"
 
 [dev-dependencies]

build.rs

@@ -1,7 +1,4 @@
-use std::env;
-use std::io::Write;
-use std::path::PathBuf;
-use std::process::Command;
+use std::{env, io::Write, path::PathBuf, process::Command};
 
 fn main() {
     let n_threads = std::thread::available_parallelism()

examples/multithreaded.rs

@@ -1,9 +1,11 @@
 //! randomx example that calculates many hashes using multiple threads
 
-use std::collections::HashMap;
-use std::sync::{Arc, RwLock};
-use std::thread;
-use std::time::Instant;
+use std::{
+    collections::HashMap,
+    sync::{Arc, RwLock},
+    thread,
+    time::Instant,
+};
 
 use anyhow::Result;
 use randomx::*;
@@ -184,10 +186,20 @@ impl RandomXFactoryInner {
 }
 
 fn main() {
-    const NUM_THREADS: u32 = 8;
+    const THREADS: usize = 8;
     // number of hashes to perform in each thread, not the total.
-    const NUM_HASHES: u32 = 10000;
+    const HASHES: usize = 10000;
+
+    // Generate each thread key
+    let mut keys: Vec<Vec<u8>> = Vec::with_capacity(THREADS);
+    let mut t = 0;
+    while t < THREADS {
+        keys.push(format!("key_{t}").as_bytes().to_vec());
+        t += 1;
+    }
 
+    println!("Initializing RandomX factory...");
+    let setup_start = Instant::now();
     // Try adding `| RandomXFlags::LARGEPAGES`.
     let mut flags = RandomXFlags::get_recommended_flags() | RandomXFlags::FULLMEM;
     if is_x86_feature_detected!("avx2") {
@@ -195,56 +207,44 @@ fn main() {
     } else if is_x86_feature_detected!("ssse3") {
         flags |= RandomXFlags::ARGON2_SSSE3;
     }
+    let factory = RandomXFactory::new_with_flags(THREADS, flags);
+    println!("Initialized RandomX factory in {:?}", setup_start.elapsed());
 
-    let factory = RandomXFactory::new_with_flags(8, flags);
-
-    let key = b"key";
-
-    let start = Instant::now();
-    let cache = RandomXCache::new(flags, &key[..]).unwrap();
-    let dataset_item_count = RandomXDataset::count().unwrap();
-    println!("Initialized RandomX cache in {:?}", start.elapsed());
-
+    println!("Starting hashing threads...");
     let mut handles = Vec::new();
-    let start = Instant::now();
-
-    for i in 0..NUM_THREADS {
+    let dataset_item_count = RandomXDataset::count().unwrap();
+    t = 0;
+    let hash_start = Instant::now();
+    while t < THREADS {
         let factory = factory.clone();
-
-        let ds_start = Instant::now();
-        let dataset = if NUM_THREADS > 1 {
-            let a = (dataset_item_count * i) / NUM_THREADS;
-            let b = (dataset_item_count * (i + 1)) / NUM_THREADS;
-            /*
-            println!("a={a}");
-            println!("b={b}");
-            println!("b-a={}", b - a);
-            */
-            RandomXDataset::new(flags, cache.clone(), a, b - a).unwrap()
-        } else {
-            RandomXDataset::new(flags, cache.clone(), 0, dataset_item_count).unwrap()
-        };
-        println!(
-            "Initialized RandomX dataset for thread #{i} in {:?}",
-            ds_start.elapsed()
-        );
-
+        let key = keys[t].clone();
         handles.push(thread::spawn(move || {
-            let key = b"key";
+            println!("Initializing RandomX cache and dataset for thread #{t}...");
+            let ds_start = Instant::now();
+            let cache = RandomXCache::new(flags, &key[..]).unwrap();
+            let dataset = RandomXDataset::new_init(flags, cache, 0, dataset_item_count).unwrap();
+            println!(
+                "Initialized RandomX cache and dataset for thread #{t} in {:?}",
+                ds_start.elapsed()
+            );
+
+            println!("Initializing RandomX VM #{t}...");
+            let vm_start = Instant::now();
             let vm = factory.create(&key[..], None, Some(dataset)).unwrap();
-            println!("Initialized RandomX VM #{i}");
-
-            let mut nonce: u32 = i;
+            println!("Initialized RandomX VM #{t} in {:?}", vm_start.elapsed());
 
-            for _ in 0..NUM_HASHES {
+            println!("Thread #{t} starts hashing...");
+            let hash_start = Instant::now();
+            for nonce in 0..(HASHES as u32) {
                 let _ = vm.calculate_hash(&nonce.to_be_bytes()[..]);
-                //println!("VM #{i} calculated hash with nonce {nonce}");
-
-                // e.g. thread 0 will use nonces 0, 8, 16, ...
-                // and thread 1 will use nonces 1, 9, 17, ...
-                nonce += NUM_THREADS;
             }
+            println!(
+                "Thread #{t} completed {} hashes in {:?}",
+                THREADS * HASHES,
+                hash_start.elapsed()
+            );
         }));
+        t += 1;
     }
 
     for handle in handles {
@@ -252,8 +252,9 @@ fn main() {
     }
 
     println!(
-        "Completed {} hashes in {:?}",
-        NUM_THREADS * NUM_HASHES,
-        start.elapsed()
+        " Hashing threads completed {} hashes in {:?}",
+        THREADS * HASHES,
+        hash_start.elapsed()
     );
+    assert_eq!(factory.get_count().unwrap(), THREADS);
 }

src/lib.rs

@@ -1,6 +1,5 @@
 #![allow(non_upper_case_globals)]
 #![allow(non_camel_case_types)]
-#![allow(non_snake_case)]
 
 //! Rust bindings to randomx, a library for computing RandomX hashes.
 //!
@@ -22,8 +21,7 @@
 
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
 
-use std::ptr;
-use std::sync::Arc;
+use std::{ptr, sync::Arc};
 
 use bitflags::bitflags;
 use libc::{c_ulong, c_void, memcpy};
@@ -137,7 +135,7 @@ impl RandomXFlags {
 
 impl Default for RandomXFlags {
     /// Default value for RandomXFlags
-    fn default() -> RandomXFlags {
+    fn default() -> Self {
         RandomXFlags::DEFAULT
     }
 }
@@ -178,7 +176,7 @@ impl RandomXCache {
     /// * ARGON2_AVX2
     ///
     /// `key` is a sequence of u8 used to initialize SuperScalarHash.
-    pub fn new(flags: RandomXFlags, key: &[u8]) -> Result<RandomXCache, RandomXError> {
+    pub fn new(flags: RandomXFlags, key: &[u8]) -> Result<Self, RandomXError> {
         if key.is_empty() {
             return Err(RandomXError::ParameterError(
                 "RandomX cache key is empty".to_string(),
@@ -191,7 +189,7 @@ impl RandomXCache {
         }
 
         let inner = RandomXCacheInner { cache_ptr };
-        let result = RandomXCache {
+        let result = Self {
             inner: Arc::new(inner),
         };
         let key_ptr = key.as_ptr() as *mut c_void;
@@ -209,7 +207,6 @@ impl RandomXCache {
 struct RandomXDatasetInner {
     dataset_ptr: *mut randomx_dataset,
     dataset_count: u32,
-    #[allow(dead_code)]
     cache: RandomXCache,
 }
 
@@ -233,8 +230,8 @@ pub struct RandomXDataset {
 }
 
 impl RandomXDataset {
-    /// Creates a new dataset object, allocates memory to the `dataset` object
-    /// and initializes it.
+    /// Creates a new dataset object and allocates memory to the
+    /// `dataset` object. Dataset must be initialized afterwards.
     ///
     /// `flags` is one of the following:
     /// * DEFAULT
@@ -242,20 +239,13 @@ impl RandomXDataset {
     ///
     /// `cache` is a cache object.
     ///
-    /// `start_item` is the item number where initialization should start,
-    /// recommended to pass in 0.
-    ///
     /// `item_count` is the total item count in the dataset, it can be
     /// retrieved with `RandomXDataset::count()`.
-    ///
-    /// Conversions may be lossy on Windows or Linux.
-    #[allow(clippy::useless_conversion)]
     pub fn new(
         flags: RandomXFlags,
         cache: RandomXCache,
-        start_item: u32,
         item_count: u32,
-    ) -> Result<RandomXDataset, RandomXError> {
+    ) -> Result<Self, RandomXError> {
         let test = unsafe { randomx_alloc_dataset(flags.bits()) };
         if test.is_null() {
             return Err(RandomXError::DatasetAllocError);
@@ -267,25 +257,63 @@ impl RandomXDataset {
             cache,
         };
 
-        let result = RandomXDataset {
+        Ok(Self {
             inner: Arc::new(inner),
-        };
-
-        /*
-        if start_item >= item_count {
-            return Err(RandomXError::DatasetAllocError);
-        }
-        */
+        })
+    }
 
+    /// Initializes the dataset object.
+    ///
+    /// `start_item` is the item number where initialization should
+    /// start.
+    ///
+    /// `item_count` is the total item count in the dataset, it can be
+    /// retrieved with `RandomXDataset::count()`.
+    pub fn init(&self, start_item: u32, item_count: u32) {
         unsafe {
             randomx_init_dataset(
-                result.inner.dataset_ptr,
-                result.inner.cache.inner.cache_ptr,
+                self.inner.dataset_ptr,
+                self.inner.cache.inner.cache_ptr,
                 c_ulong::from(start_item),
                 c_ulong::from(item_count),
             );
         }
-        Ok(result)
+    }
+
+    /// Creates a new dataset object, allocates memory to the `dataset` object
+    /// and initializes it.
+    ///
+    /// `flags` is one of the following:
+    /// * DEFAULT
+    /// * LARGEPAGES
+    ///
+    /// `cache` is a cache object.
+    ///
+    /// `start_item` is the item number where initialization should start,
+    /// recommended to pass in 0.
+    ///
+    /// `item_count` is the total item count in the dataset, it can be
+    /// retrieved with `RandomXDataset::count()`.
+    pub fn new_init(
+        flags: RandomXFlags,
+        cache: RandomXCache,
+        start_item: u32,
+        item_count: u32,
+    ) -> Result<Self, RandomXError> {
+        let dataset = Self::new(flags, cache, item_count)?;
+        dataset.init(start_item, item_count);
+        Ok(dataset)
+    }
+
+    /// Creates a new dataset subset and initializes it.
+    ///
+    /// `start_item` is the item number where initialization should start.
+    ///
+    /// `item_count` is the total item count in the subset.
+    pub fn subset_init(&self, start_item: u32, item_count: u32) -> Self {
+        let subset = self.clone();
+        subset.init(start_item, item_count);
+        subset
     }
 
     /// Returns the number of items in the `dataset` or an error on failure.
@@ -370,7 +398,7 @@ impl RandomXVM {
         flags: RandomXFlags,
         cache: Option<RandomXCache>,
         dataset: Option<RandomXDataset>,
-    ) -> Result<RandomXVM, RandomXError> {
+    ) -> Result<Self, RandomXError> {
         let is_full_mem = flags.contains(RandomXFlags::FULLMEM);
 
         match (cache, dataset) {
@@ -389,7 +417,7 @@ impl RandomXVM {
 
                 let vm = unsafe { randomx_create_vm(flags.bits(), cache_ptr, dataset_ptr) };
 
-                Ok(RandomXVM {
+                Ok(Self {
                     vm,
                     flags,
                     linked_cache: cache,
@@ -486,9 +514,8 @@ impl RandomXVM {
 
         // Not len() as last iteration assigns final hash
         let iterations = input.len() + 1;
-
-        #[allow(clippy::needless_range_loop)]
-        for i in 0..iterations {
+        let mut i = 0;
+        while i < iterations {
             if i == iterations - 1 {
                 // For last iteration
                 unsafe {
@@ -533,6 +560,8 @@ impl RandomXVM {
                 let output: Vec<u8> = arr.to_vec();
                 result.push(output);
             }
+
+            i += 1;
         }
 
         Ok(result)
@@ -541,7 +570,7 @@ impl RandomXVM {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use crate::{RandomXCache, RandomXDataset, RandomXFlags, RandomXVM};
 
     #[test]
     fn lib_alloc_cache() {
@@ -557,7 +586,7 @@ mod tests {
         let key = "Key";
         let cache = RandomXCache::new(flags, key.as_bytes()).unwrap();
         let dataset =
-            RandomXDataset::new(flags, cache.clone(), 0, RandomXDataset::count().unwrap())
+            RandomXDataset::new_init(flags, cache.clone(), 0, RandomXDataset::count().unwrap())
                 .expect("Failed to allocate dataset");
         drop(dataset);
         drop(cache);
@@ -572,7 +601,8 @@ mod tests {
             RandomXVM::new(flags, Some(cache.clone()), None).expect("Failed to allocate VM");
         drop(vm);
         let dataset =
-            RandomXDataset::new(flags, cache.clone(), 0, RandomXDataset::count().unwrap()).unwrap();
+            RandomXDataset::new_init(flags, cache.clone(), 0, RandomXDataset::count().unwrap())
+                .unwrap();
         vm = RandomXVM::new(flags, Some(cache.clone()), Some(dataset.clone()))
             .expect("Failed to allocate VM");
         drop(dataset);
@@ -586,7 +616,8 @@ mod tests {
         let key = "Key";
         let cache = RandomXCache::new(flags, key.as_bytes()).unwrap();
         let dataset =
-            RandomXDataset::new(flags, cache.clone(), 0, RandomXDataset::count().unwrap()).unwrap();
+            RandomXDataset::new_init(flags, cache.clone(), 0, RandomXDataset::count().unwrap())
+                .unwrap();
         let memory = dataset.get_data().unwrap_or_else(|_| Vec::new());
         assert!(!memory.is_empty(), "Failed to get dataset memory");
         let v = vec![0u8; memory.len()];
@@ -619,7 +650,7 @@ mod tests {
 
         let cache3 = RandomXCache::new(flags, key.as_bytes()).unwrap();
         let dataset3 =
-            RandomXDataset::new(flags, cache3.clone(), 0, RandomXDataset::count().unwrap())
+            RandomXDataset::new_init(flags, cache3.clone(), 0, RandomXDataset::count().unwrap())
                 .unwrap();
         let mut vm3 = RandomXVM::new(flags2, None, Some(dataset3.clone())).unwrap();
         let hash4 = vm3.calculate_hash(input.as_bytes()).expect("no data");
@@ -631,7 +662,7 @@ mod tests {
 
         let cache4 = RandomXCache::new(flags, key.as_bytes()).unwrap();
         let dataset4 =
-            RandomXDataset::new(flags, cache4.clone(), 0, RandomXDataset::count().unwrap())
+            RandomXDataset::new_init(flags, cache4.clone(), 0, RandomXDataset::count().unwrap())
                 .unwrap();
         let vm4 = RandomXVM::new(flags2, Some(cache4), Some(dataset4.clone())).unwrap();
         let hash6 = vm3.calculate_hash(input.as_bytes()).expect("no data");
@@ -681,7 +712,8 @@ mod tests {
         let input = "Input";
         let cache = RandomXCache::new(flags, key.as_bytes()).unwrap();
         let dataset =
-            RandomXDataset::new(flags, cache.clone(), 0, RandomXDataset::count().unwrap()).unwrap();
+            RandomXDataset::new_init(flags, cache.clone(), 0, RandomXDataset::count().unwrap())
+                .unwrap();
         let vm = RandomXVM::new(flags, Some(cache.clone()), Some(dataset.clone())).unwrap();
         let hash = vm.calculate_hash(input.as_bytes()).expect("no data");
         assert_eq!(
@@ -697,7 +729,7 @@ mod tests {
 
         let cache1 = RandomXCache::new(flags, key.as_bytes()).unwrap();
         let dataset1 =
-            RandomXDataset::new(flags, cache1.clone(), 0, RandomXDataset::count().unwrap())
+            RandomXDataset::new_init(flags, cache1.clone(), 0, RandomXDataset::count().unwrap())
                 .unwrap();
         let vm1 = RandomXVM::new(flags, Some(cache1.clone()), Some(dataset1.clone())).unwrap();
         let hash1 = vm1.calculate_hash(input.as_bytes()).expect("no data");
@@ -720,7 +752,8 @@ mod tests {
         let input = "Input";
         let cache = RandomXCache::new(flags, key.as_bytes()).unwrap();
         let dataset =
-            RandomXDataset::new(flags, cache.clone(), 0, RandomXDataset::count().unwrap()).unwrap();
+            RandomXDataset::new_init(flags, cache.clone(), 0, RandomXDataset::count().unwrap())
+                .unwrap();
         let vm = RandomXVM::new(flags, Some(cache.clone()), Some(dataset.clone())).unwrap();
         drop(dataset);
         drop(cache);
@@ -736,7 +769,7 @@ mod tests {
 
         let cache1 = RandomXCache::new(flags, key.as_bytes()).unwrap();
         let dataset1 =
-            RandomXDataset::new(flags, cache1.clone(), 0, RandomXDataset::count().unwrap())
+            RandomXDataset::new_init(flags, cache1.clone(), 0, RandomXDataset::count().unwrap())
                 .unwrap();
         let vm1 = RandomXVM::new(flags, Some(cache1.clone()), Some(dataset1.clone())).unwrap();
         drop(dataset1);
@@ -761,7 +794,7 @@ mod tests {
         let cache = RandomXCache::new(flags, key).unwrap();
 
         let dataset =
-            RandomXDataset::new(flags, cache, 0, RandomXDataset::count().unwrap()).unwrap();
+            RandomXDataset::new_init(flags, cache, 0, RandomXDataset::count().unwrap()).unwrap();
 
         let fast_vm = RandomXVM::new(flags, None, Some(dataset)).unwrap();
 
@@ -797,7 +830,7 @@ mod tests {
         let cache = RandomXCache::new(flags, key).unwrap();
 
         let dataset =
-            RandomXDataset::new(flags, cache, 0, RandomXDataset::count().unwrap()).unwrap();
+            RandomXDataset::new_init(flags, cache, 0, RandomXDataset::count().unwrap()).unwrap();
 
         let vm = RandomXVM::new(flags, None, Some(dataset)).unwrap();