research/pow: Add difficulty adjustment algorithm with passing Monero tests.

script/research/pow/Cargo.toml

@@ -11,5 +11,9 @@ darkfi-serial = {path = "../../../src/serial"}
 darkfi-sdk = {path = "../../../src/sdk", features = ["async"]}
 darkfi = {path = "../../../", features = ["util"]}
 
-blake2b_simd = "1.0.1"
 rand = "0.8.5"
+blake2b_simd = "1.0.1"
+num-bigint = "0.4.4"
+
+[patch.crates-io]
+blake2b_simd = {git = "https://github.com/parazyd/blake2_simd", branch = "impl-common"}

script/research/pow/gen_wide_data.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# Copyright (c) 2014-2023, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+import random
+
+DIFFICULTY_TARGET = 120
+DIFFICULTY_WINDOW = 720
+DIFFICULTY_LAG = 15
+DIFFICULTY_CUT = 60
+
+
+def difficulty():
+    times = []
+    diffs = []
+    while True:
+        if len(times) <= 1:
+            diff = 1
+        else:
+            begin = max(len(times) - DIFFICULTY_WINDOW - DIFFICULTY_LAG, 0)
+            end = min(begin + DIFFICULTY_WINDOW, len(times))
+            length = end - begin
+            assert length >= 2
+            if length <= DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT:
+                cut_begin = 0
+                cut_end = length
+            else:
+                excess = length - (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT)
+                cut_begin = (excess + 1) // 2
+                cut_end = length - excess // 2
+            assert cut_begin + 2 <= cut_end
+            wnd = times[begin:end]
+            wnd.sort()
+            dtime = wnd[cut_end - 1] - wnd[cut_begin]
+            dtime = max(dtime, 1)
+            ddiff = sum(diffs[begin + cut_begin + 1:begin + cut_end])
+            diff = (ddiff * DIFFICULTY_TARGET + dtime - 1) // dtime
+        times.append((yield diff))
+        diffs.append(diff)
+
+
+random.seed(1)
+time = 1000
+gen = difficulty()
+diff = next(gen)
+for i in range(100000):
+    power = 100 if i < 10000 else 100000000 if i < 500 else 1000000000000 if i < 1000 else 1000000000000000 if i < 2000 else 10000000000000000000 if i < 4000 else 1000000000000000000000000
+    time += random.randint(-diff // power - 10, 3 * diff // power + 10)
+    print(time, diff)
+    diff = gen.send(time)

script/research/pow/src/main.rs

@@ -1,6 +1,7 @@
 /* This file is part of DarkFi (https://dark.fi)
  *
  * Copyright (C) 2020-2023 Dyne.org foundation
+ * Copyright (C) 2014-2023 The Monero Project (Under MIT license)
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
@@ -17,6 +18,7 @@
  */
 
 use std::{
+    cmp::min,
     sync::{
         atomic::{AtomicBool, AtomicU32, Ordering},
         Arc,
@@ -28,20 +30,31 @@ use std::{
 use darkfi::{util::time::Timestamp, Result};
 use darkfi_sdk::{
     crypto::MerkleTree,
+    num_traits::{One, Zero},
     pasta::{group::ff::FromUniformBytes, pallas},
 };
 use darkfi_serial::{async_trait, Encodable, SerialDecodable, SerialEncodable};
+use num_bigint::BigUint;
 use rand::{rngs::OsRng, Rng};
 use randomx::{RandomXCache, RandomXDataset, RandomXFlags, RandomXVM};
 
+#[cfg(test)]
+mod tests;
+
+// Number of threads to use for hashing
+const NUM_THREADS: u32 = 4;
 /// Constant genesis block string used as the previous block hash
 const GENESIS: &[u8] = b"genesis";
-/// The target mining difficulty
-const DIFFICULTY: usize = 1;
 /// The output length of the BLAKE2b hash in bytes
 const HASH_LEN: usize = 32;
-/// The amount of blocks the main loop will mine until the program exits
-const N_BLOCKS: usize = 5;
+/// Blocks, must be >=2
+const DIFFICULTY_WINDOW: usize = 720;
+/// Timestamps to cut after sorting, (2*DIFFICULTY_CUT<=DIFFICULTY_WINDOW-2) must be true
+const DIFFICULTY_CUT: usize = 60;
+/// !!!
+const DIFFICULTY_LAG: usize = 15;
+/// Target block time in seconds
+const DIFFICULTY_TARGET: usize = 60;
 
 #[derive(Clone, SerialEncodable, SerialDecodable)]
 struct Transaction(Vec<u8>);
@@ -92,6 +105,49 @@ impl Block {
     }
 }
 
+fn next_difficulty(
+    mut timestamps: Vec<u64>,
+    mut cummulative_difficulties: Vec<BigUint>,
+    target_seconds: usize,
+) -> BigUint {
+    if timestamps.len() > DIFFICULTY_WINDOW {
+        timestamps.resize(DIFFICULTY_WINDOW, 1);
+        cummulative_difficulties.resize(DIFFICULTY_WINDOW, BigUint::one());
+    }
+
+    let length = timestamps.len();
+    assert!(length == cummulative_difficulties.len());
+
+    if length <= 1 {
+        return BigUint::one()
+    }
+
+    assert!(length <= DIFFICULTY_WINDOW);
+    timestamps.sort_unstable();
+
+    let cut_begin: usize;
+    let cut_end: usize;
+
+    if length <= DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT {
+        cut_begin = 0;
+        cut_end = length;
+    } else {
+        cut_begin = (length - (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT) + 1) / 2;
+        cut_end = cut_begin + (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT);
+    }
+
+    assert!(/* cut_begin >= 0 && */ cut_begin + 2 <= cut_end && cut_end <= length);
+
+    let mut time_span = timestamps[cut_end - 1] - timestamps[cut_begin];
+    if time_span == 0 {
+        time_span = 1;
+    }
+
+    let total_work = &cummulative_difficulties[cut_end - 1] - &cummulative_difficulties[cut_begin];
+    assert!(total_work >= BigUint::zero());
+    (total_work * target_seconds + time_span - BigUint::one()) / time_span
+}
+
 fn main() -> Result<()> {
     // Construct the genesis block
     let genesis_hash =
@@ -101,7 +157,7 @@ fn main() -> Result<()> {
         header: BlockHeader {
             nonce: 0,
             previous_hash: genesis_hash,
-            timestamp: Timestamp(1693213806),
+            timestamp: Timestamp::current_time(),
             txtree: MerkleTree::new(100),
         },
         transactions: vec![],
@@ -110,15 +166,47 @@ fn main() -> Result<()> {
     let genesis_tx = Transaction(vec![1, 3, 3, 7]);
     genesis_block.insert_tx(&genesis_tx)?;
 
+    let mut blockchain = vec![genesis_block.clone()];
+    let mut cummulative_difficulties = vec![BigUint::one()];
+
+    let mut cummulative_difficulty = BigUint::one();
+
+    // Melt the CPU
+    let mut n = 1;
     let mut cur_block = genesis_block;
-    for i in 0..N_BLOCKS {
+    loop {
+        // Calculate the next difficulty
+        let begin: usize;
+        let end: usize;
+        if n < DIFFICULTY_WINDOW + DIFFICULTY_LAG {
+            begin = 0;
+            end = min(n, DIFFICULTY_WINDOW);
+        } else {
+            end = n - DIFFICULTY_LAG;
+            begin = end - DIFFICULTY_WINDOW;
+        }
+
+        let timestamps: Vec<u64> =
+            blockchain[begin..end].iter().map(|x| x.header.timestamp.0).collect();
+        let difficulties: Vec<BigUint> = cummulative_difficulties[begin..end].to_vec();
+
+        let difficulty = next_difficulty(timestamps, difficulties, DIFFICULTY_TARGET);
+
+        let target = BigUint::from_bytes_be(&[
+            0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+            0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+            0xff, 0xff, 0xff, 0xff,
+        ]) / &difficulty;
+        println!("[{}] [MINER] Difficulty: {}", n, difficulty);
+        println!("[{}] [MINER] Target: {}", n, target);
+
         // Get the PoW input. The key changes with every mined block.
         let pow_input = cur_block.hash()?;
-        println!("[{}] [MINER] PoW Input: {}", i, pow_input.to_hex());
+        println!("[{}] [MINER] PoW Input: {}", n, pow_input.to_hex());
 
         let miner_setup = Instant::now();
         let flags = RandomXFlags::default() | RandomXFlags::FULLMEM;
-        println!("[{}] [MINER] Initializing RandomX dataset...", i);
+        println!("[{}] [MINER] Initializing RandomX dataset...", n);
         let dataset = Arc::new(RandomXDataset::new(flags, pow_input.as_bytes(), 1).unwrap());
 
         // The miner creates a block
@@ -135,47 +223,40 @@ fn main() -> Result<()> {
         let tx1 = Transaction(OsRng.gen::<[u8; 32]>().to_vec());
         miner_block.insert_tx(&tx0)?;
         miner_block.insert_tx(&tx1)?;
-        println!("[{}] [MINER] Setup time: {:?}", i, miner_setup.elapsed());
+        println!("[{}] [MINER] Setup time: {:?}", n, miner_setup.elapsed());
 
         // Multithreaded mining setup
         let mining_time = Instant::now();
-        // Let's use 4 threads
-        const NUM_THREADS: u32 = 4;
         let mut handles = vec![];
         let found_block = Arc::new(AtomicBool::new(false));
         let found_nonce = Arc::new(AtomicU32::new(0));
         for t in 0..NUM_THREADS {
+            let target = target.clone();
             let mut block = miner_block.clone();
             let found_block = Arc::clone(&found_block);
             let found_nonce = Arc::clone(&found_nonce);
             let dataset = dataset.clone();
             handles.push(thread::spawn(move || {
-                println!("[{}] [MINER] Initializing RandomX VM #{}...", i, t);
+                println!("[{}] [MINER] Initializing RandomX VM #{}...", n, t);
                 block.header.nonce = t;
                 let vm = RandomXVM::new_fast(flags, &dataset).unwrap();
                 loop {
                     if found_block.load(Ordering::SeqCst) {
-                        println!("[{}] [MINER] Block was found, thread #{} exiting", i, t);
+                        println!("[{}] [MINER] Block was found, thread #{} exiting", n, t);
                         break
                     }
 
                     let out_hash = vm.hash(block.hash().unwrap().as_bytes());
-                    let mut success = true;
-                    for idx in 0..DIFFICULTY {
-                        if out_hash[idx] != 0x00 {
-                            success = false;
-                        }
-                    }
-
-                    if success {
+                    let out_hash = BigUint::from_bytes_be(&out_hash);
+                    if out_hash <= target {
                         found_block.store(true, Ordering::SeqCst);
                         found_nonce.store(block.header.nonce, Ordering::SeqCst);
                         println!(
                             "[{}] [MINER] Thread #{} found block using nonce {}",
-                            i, t, block.header.nonce
+                            n, t, block.header.nonce
                         );
-                        println!("[{}] [MINER] Block hash {}", i, block.hash().unwrap().to_hex(),);
-                        println!("[{}] [MINER] RandomX hash bytes: {:?}", i, out_hash);
+                        println!("[{}] [MINER] Block hash {}", n, block.hash().unwrap().to_hex(),);
+                        println!("[{}] [MINER] RandomX hash bytes: {:?}", n, out_hash);
                         break
                     }
 
@@ -186,11 +267,10 @@ fn main() -> Result<()> {
             }))
         }
 
-        // Melt the CPU
         for handle in handles {
             let _ = handle.join();
         }
-        println!("[{}] [MINER] Mining time: {:?}", i, mining_time.elapsed());
+        println!("[{}] [MINER] Mining time: {:?}", n, mining_time.elapsed());
 
         // Set the valid mined nonce in the block that's broadcasted
         miner_block.header.nonce = found_nonce.load(Ordering::SeqCst);
@@ -200,18 +280,20 @@ fn main() -> Result<()> {
         let flags = RandomXFlags::default();
         let cache = RandomXCache::new(flags, pow_input.as_bytes()).unwrap();
         let vm = RandomXVM::new(flags, &cache).unwrap();
-        println!("[{}] [VERIFIER] Setup time: {:?}", i, verifier_setup.elapsed());
+        println!("[{}] [VERIFIER] Setup time: {:?}", n, verifier_setup.elapsed());
 
         let verification_time = Instant::now();
         let out_hash = vm.hash(miner_block.hash()?.as_bytes());
-        for idx in 0..DIFFICULTY {
-            assert!(out_hash[idx] == 0x00);
-        }
-        println!("[{}] [VERIFIER] Verification time: {:?}", i, verification_time.elapsed());
+        let out_hash = BigUint::from_bytes_be(&out_hash);
+        assert!(out_hash <= target);
+        println!("[{}] [VERIFIER] Verification time: {:?}", n, verification_time.elapsed());
 
         // The new block appends to the blockchain
-        cur_block = miner_block;
-    }
+        cur_block = miner_block.clone();
+        blockchain.push(miner_block);
+        cummulative_difficulty += difficulty;
+        cummulative_difficulties.push(cummulative_difficulty.clone());
 
-    Ok(())
+        n += 1;
+    }
 }

script/research/pow/src/tests.rs

@@ -0,0 +1,56 @@
+use std::{
+    cmp::min,
+    io::{BufRead, Cursor},
+    process::Command,
+};
+
+use darkfi_sdk::num_traits::{Num, Zero};
+use num_bigint::BigUint;
+
+use crate::{next_difficulty, DIFFICULTY_LAG, DIFFICULTY_WINDOW};
+
+const DEFAULT_TEST_DIFFICULTY_TARGET: usize = 120;
+
+#[test]
+fn test_wide_difficulty() {
+    let mut timestamps: Vec<u64> = vec![];
+    let mut cummulative_difficulties: Vec<BigUint> = vec![];
+    let mut cummulative_difficulty = BigUint::zero();
+
+    let output = Command::new("./gen_wide_data.py").output().unwrap();
+    let reader = Cursor::new(output.stdout);
+
+    for (n, line) in reader.lines().enumerate() {
+        let line = line.unwrap();
+        let parts: Vec<String> = line.split(' ').map(|x| x.to_string()).collect();
+        assert!(parts.len() == 2);
+
+        let timestamp = parts[0].parse::<u64>().unwrap();
+        let difficulty = BigUint::from_str_radix(&parts[1], 10).unwrap();
+
+        let begin: usize;
+        let end: usize;
+        if n < DIFFICULTY_WINDOW + DIFFICULTY_LAG {
+            begin = 0;
+            end = min(n, DIFFICULTY_WINDOW);
+        } else {
+            end = n - DIFFICULTY_LAG;
+            begin = end - DIFFICULTY_WINDOW;
+        }
+
+        let timestamps_cut = timestamps[begin..end].to_vec();
+        let difficulty_cut = cummulative_difficulties[begin..end].to_vec();
+        let res = next_difficulty(timestamps_cut, difficulty_cut, DEFAULT_TEST_DIFFICULTY_TARGET);
+
+        if res != difficulty {
+            eprintln!("Wrong wide difficulty for block {}", n);
+            eprintln!("Expected: {}", difficulty);
+            eprintln!("Found: {}", res);
+            assert!(res == difficulty);
+        }
+
+        timestamps.push(timestamp);
+        cummulative_difficulty += difficulty;
+        cummulative_difficulties.push(cummulative_difficulty.clone());
+    }
+}