validator: permanently store ranks as blockchain expands

bin/darkfid/src/tests/forks.rs

@@ -17,7 +17,7 @@
  */
 
 use darkfi::{
-    blockchain::Blockchain,
+    blockchain::{BlockInfo, Blockchain},
     validator::{consensus::Fork, pow::PoWModule},
     Result,
 };
@@ -26,38 +26,55 @@ use darkfi::{
 fn forks() -> Result<()> {
     smol::block_on(async {
         // Dummy records we will insert
-        let record0 = blake3::hash(b"Let there be dark!");
-        let record1 = blake3::hash(b"Never skip brain day.");
+        let record1 = blake3::hash(b"Let there be dark!");
+        let record2 = blake3::hash(b"Never skip brain day.");
 
         // Create a temporary blockchain and a PoW module
         let blockchain = Blockchain::new(&sled::Config::new().temporary(true).open()?)?;
         let module = PoWModule::new(blockchain.clone(), 90, None)?;
 
+        // Generate and insert default genesis block
+        let genesis_block = BlockInfo::default();
+        blockchain.add_block(&genesis_block)?;
+        let genesis_block_hash = genesis_block.hash()?;
+
         // Create a fork
         let fork = Fork::new(blockchain.clone(), module).await?;
 
         // Add a dummy record to fork
-        fork.overlay.lock().unwrap().order.insert(&[0], &[record0])?;
+        fork.overlay.lock().unwrap().order.insert(&[1], &[record1])?;
 
         // Verify blockchain doesn't contain the record
-        assert_eq!(blockchain.order.get(&[0], false)?, [None]);
-        assert_eq!(fork.overlay.lock().unwrap().order.get(&[0], true)?, [Some(record0)]);
+        assert_eq!(blockchain.order.get(&[0, 1], false)?, [Some(genesis_block_hash), None]);
+        assert_eq!(
+            fork.overlay.lock().unwrap().order.get(&[0, 1], true)?,
+            [Some(genesis_block_hash), Some(record1)]
+        );
 
         // Now we are going to clone the fork
         let fork_clone = fork.full_clone()?;
 
-        // Verify it cointains the original record
-        assert_eq!(fork_clone.overlay.lock().unwrap().order.get(&[0], true)?, [Some(record0)]);
+        // Verify it contains the original records
+        assert_eq!(
+            fork_clone.overlay.lock().unwrap().order.get(&[0, 1], true)?,
+            [Some(genesis_block_hash), Some(record1)]
+        );
 
         // Add another dummy record to cloned fork
-        fork_clone.overlay.lock().unwrap().order.insert(&[1], &[record1])?;
+        fork_clone.overlay.lock().unwrap().order.insert(&[2], &[record2])?;
 
         // Verify blockchain and original fork don't contain the second record
-        assert_eq!(blockchain.order.get(&[0, 1], false)?, [None, None]);
-        assert_eq!(fork.overlay.lock().unwrap().order.get(&[0, 1], false)?, [Some(record0), None]);
         assert_eq!(
-            fork_clone.overlay.lock().unwrap().order.get(&[0, 1], true)?,
-            [Some(record0), Some(record1)]
+            blockchain.order.get(&[0, 1, 2], false)?,
+            [Some(genesis_block_hash), None, None]
+        );
+        assert_eq!(
+            fork.overlay.lock().unwrap().order.get(&[0, 1, 2], false)?,
+            [Some(genesis_block_hash), Some(record1), None]
+        );
+        assert_eq!(
+            fork_clone.overlay.lock().unwrap().order.get(&[0, 1, 2], true)?,
+            [Some(genesis_block_hash), Some(record1), Some(record2)]
         );
 
         Ok(())

doc/src/arch/consensus.md

@@ -78,9 +78,9 @@ Proof of Work algorithm lowers the difficulty target as hashpower grows.
 This means that blocks will have to be mined for a lower target, therefore
 rank higher, as they go further away from `MAX_INT`.
 
-Similar to blocks, forks rank is a tuple, with the first part being the
+Similar to blocks, blockchain/forks rank is a tuple, with the first part being the
 sum of its block's squared target distances, and the second being the sum of
-their squared hash distances Squared distances are used to disproportionately
+their squared hash distances. Squared distances are used to disproportionately
 favors smaller targets, with the idea being that it will be harder to trigger
 a longer reorg between forks. When we compare forks, we first check the first
 sum, and if its tied, we use the second as the tie breaker, since we know it

src/blockchain/block_store.rs

@@ -490,6 +490,61 @@ impl BlockOrderStoreOverlay {
     }
 }
 
+/// Auxiliary structure used to keep track of block ranking information.
+/// Note: we only need height cummulative ranks, but we also keep its actual
+/// ranks, so we can verify the sequence and/or know specific block height
+/// ranks, if ever needed.
+#[derive(Debug)]
+pub struct BlockRanks {
+    /// Block target rank
+    pub target_rank: BigUint,
+    /// Height cummulative targets rank
+    pub targets_rank: BigUint,
+    /// Block hash rank
+    pub hash_rank: BigUint,
+    /// Height cummulative hashes rank
+    pub hashes_rank: BigUint,
+}
+
+impl BlockRanks {
+    pub fn new(
+        target_rank: BigUint,
+        targets_rank: BigUint,
+        hash_rank: BigUint,
+        hashes_rank: BigUint,
+    ) -> Self {
+        Self { target_rank, targets_rank, hash_rank, hashes_rank }
+    }
+}
+
+// Note: Doing all the imports here as this might get obselete if
+// we implemented Encodable/Decodable for num_bigint::BigUint.
+impl darkfi_serial::Encodable for BlockRanks {
+    fn encode<S: std::io::Write>(&self, mut s: S) -> std::io::Result<usize> {
+        let mut len = 0;
+        len += self.target_rank.to_bytes_be().encode(&mut s)?;
+        len += self.targets_rank.to_bytes_be().encode(&mut s)?;
+        len += self.hash_rank.to_bytes_be().encode(&mut s)?;
+        len += self.hashes_rank.to_bytes_be().encode(&mut s)?;
+        Ok(len)
+    }
+}
+
+impl darkfi_serial::Decodable for BlockRanks {
+    fn decode<D: std::io::Read>(mut d: D) -> std::io::Result<Self> {
+        let bytes: Vec<u8> = darkfi_serial::Decodable::decode(&mut d)?;
+        let target_rank: BigUint = BigUint::from_bytes_be(&bytes);
+        let bytes: Vec<u8> = darkfi_serial::Decodable::decode(&mut d)?;
+        let targets_rank: BigUint = BigUint::from_bytes_be(&bytes);
+        let bytes: Vec<u8> = darkfi_serial::Decodable::decode(&mut d)?;
+        let hash_rank: BigUint = BigUint::from_bytes_be(&bytes);
+        let bytes: Vec<u8> = darkfi_serial::Decodable::decode(&mut d)?;
+        let hashes_rank: BigUint = BigUint::from_bytes_be(&bytes);
+        let ret = Self { target_rank, targets_rank, hash_rank, hashes_rank };
+        Ok(ret)
+    }
+}
+
 /// Auxiliary structure used to keep track of block PoW difficulty information.
 /// Note: we only need height cummulative difficulty, but we also keep its actual
 /// difficulty, so we can verify the sequence and/or know specific block height
@@ -504,6 +559,8 @@ pub struct BlockDifficulty {
     pub difficulty: BigUint,
     /// Height cummulative difficulty (total + height difficulty)
     pub cummulative_difficulty: BigUint,
+    /// Block ranks
+    pub ranks: BlockRanks,
 }
 
 impl BlockDifficulty {
@@ -512,8 +569,20 @@ impl BlockDifficulty {
         timestamp: Timestamp,
         difficulty: BigUint,
         cummulative_difficulty: BigUint,
+        ranks: BlockRanks,
     ) -> Self {
-        Self { height, timestamp, difficulty, cummulative_difficulty }
+        Self { height, timestamp, difficulty, cummulative_difficulty, ranks }
+    }
+
+    /// Represents the genesis block difficulty
+    pub fn genesis(timestamp: Timestamp) -> Self {
+        let ranks = BlockRanks::new(
+            BigUint::from(0u64),
+            BigUint::from(0u64),
+            BigUint::from(0u64),
+            BigUint::from(0u64),
+        );
+        BlockDifficulty::new(0, timestamp, BigUint::from(0u64), BigUint::from(0u64), ranks)
     }
 }
 
@@ -526,6 +595,7 @@ impl darkfi_serial::Encodable for BlockDifficulty {
         len += self.timestamp.encode(&mut s)?;
         len += self.difficulty.to_bytes_be().encode(&mut s)?;
         len += self.cummulative_difficulty.to_bytes_be().encode(&mut s)?;
+        len += self.ranks.encode(&mut s)?;
         Ok(len)
     }
 }
@@ -538,7 +608,8 @@ impl darkfi_serial::Decodable for BlockDifficulty {
         let difficulty: BigUint = BigUint::from_bytes_be(&bytes);
         let bytes: Vec<u8> = darkfi_serial::Decodable::decode(&mut d)?;
         let cummulative_difficulty: BigUint = BigUint::from_bytes_be(&bytes);
-        let ret = Self { height, timestamp, difficulty, cummulative_difficulty };
+        let ranks: BlockRanks = darkfi_serial::Decodable::decode(&mut d)?;
+        let ret = Self { height, timestamp, difficulty, cummulative_difficulty, ranks };
         Ok(ret)
     }
 }
@@ -605,6 +676,15 @@ impl BlockDifficultyStore {
         Ok(ret)
     }
 
+    /// Fetch the last record in the tree, based on the `Ord`
+    /// implementation for `Vec<u8>`. If the tree is empty,
+    /// returns `None`.
+    pub fn get_last(&self) -> Result<Option<BlockDifficulty>> {
+        let Some(found) = self.0.last()? else { return Ok(None) };
+        let block_difficulty = deserialize(&found.1)?;
+        Ok(Some(block_difficulty))
+    }
+
     /// Fetch the last N records from the block difficulties store, in order.
     pub fn get_last_n(&self, n: usize) -> Result<Vec<BlockDifficulty>> {
         // Build an iterator to retrieve last N records

src/blockchain/mod.rs

@@ -28,8 +28,8 @@ use crate::{tx::Transaction, Error, Result};
 /// Block related definitions and storage implementations
 pub mod block_store;
 pub use block_store::{
-    Block, BlockDifficultyStore, BlockDifficultyStoreOverlay, BlockInfo, BlockOrderStore,
-    BlockOrderStoreOverlay, BlockStore, BlockStoreOverlay,
+    Block, BlockDifficulty, BlockDifficultyStore, BlockDifficultyStoreOverlay, BlockInfo,
+    BlockOrderStore, BlockOrderStoreOverlay, BlockStore, BlockStoreOverlay,
 };
 
 /// Header definition and storage implementation
@@ -223,6 +223,12 @@ impl Blockchain {
         self.order.get_first()
     }
 
+    /// Retrieve genesis (first) block info.
+    pub fn genesis_block(&self) -> Result<BlockInfo> {
+        let (_, hash) = self.genesis()?;
+        Ok(self.get_blocks_by_hash(&[hash])?[0].clone())
+    }
+
     /// Retrieve the last block height and hash.
     pub fn last(&self) -> Result<(u64, blake3::Hash)> {
         self.order.get_last()
@@ -234,6 +240,17 @@ impl Blockchain {
         Ok(self.get_blocks_by_hash(&[hash])?[0].clone())
     }
 
+    /// Retrieve the last block difficulty. If the tree is empty,
+    /// returns `BlockDifficulty::genesis` difficulty.
+    pub fn last_block_difficulty(&self) -> Result<BlockDifficulty> {
+        if let Some(found) = self.difficulties.get_last()? {
+            return Ok(found)
+        }
+
+        let genesis_block = self.genesis_block()?;
+        Ok(BlockDifficulty::genesis(genesis_block.header.timestamp))
+    }
+
     /// Check if block order for the given height is in the database.
     pub fn has_height(&self, height: u64) -> Result<bool> {
         let vec = match self.order.get(&[height], true) {

src/validator/consensus.rs

@@ -27,8 +27,8 @@ use smol::lock::RwLock;
 
 use crate::{
     blockchain::{
-        block_store::BlockDifficulty, BlockInfo, Blockchain, BlockchainOverlay,
-        BlockchainOverlayPtr, Header,
+        block_store::{BlockDifficulty, BlockRanks},
+        BlockInfo, Blockchain, BlockchainOverlay, BlockchainOverlayPtr, Header,
     },
     tx::Transaction,
     util::time::Timestamp,
@@ -478,6 +478,10 @@ impl Fork {
         let mempool =
             blockchain.get_pending_txs()?.iter().map(|tx| blake3::hash(&serialize(tx))).collect();
         let overlay = BlockchainOverlay::new(&blockchain)?;
+        // Retrieve last block difficulty to access current ranks
+        let last_difficulty = blockchain.last_block_difficulty()?;
+        let targets_rank = last_difficulty.ranks.targets_rank;
+        let hashes_rank = last_difficulty.ranks.hashes_rank;
         Ok(Self {
             blockchain,
             overlay,
@@ -485,8 +489,8 @@ impl Fork {
             proposals: vec![],
             diffs: vec![],
             mempool,
-            targets_rank: BigUint::from(0u64),
-            hashes_rank: BigUint::from(0u64),
+            targets_rank,
+            hashes_rank,
         })
     }
 
@@ -542,21 +546,28 @@ impl Fork {
         // Calculate block rank
         let (target_distance_sq, hash_distance_sq) = block_rank(&proposal.block, &next_target)?;
 
+        // Update fork ranks
+        self.targets_rank += target_distance_sq.clone();
+        self.hashes_rank += hash_distance_sq.clone();
+
         // Generate block difficulty and update PoW module
         let cummulative_difficulty =
             self.module.cummulative_difficulty.clone() + next_difficulty.clone();
+        let ranks = BlockRanks::new(
+            target_distance_sq,
+            self.targets_rank.clone(),
+            hash_distance_sq,
+            self.hashes_rank.clone(),
+        );
         let block_difficulty = BlockDifficulty::new(
             proposal.block.header.height,
             proposal.block.header.timestamp,
             next_difficulty,
             cummulative_difficulty,
+            ranks,
         );
         self.module.append_difficulty(&self.overlay, block_difficulty)?;
 
-        // Update fork ranks
-        self.targets_rank += target_distance_sq;
-        self.hashes_rank += hash_distance_sq;
-
         // Push proposal's hash
         self.proposals.push(proposal.hash);
 

src/validator/mod.rs

@@ -26,7 +26,7 @@ use smol::lock::RwLock;
 
 use crate::{
     blockchain::{
-        block_store::{BlockDifficulty, BlockInfo},
+        block_store::{BlockDifficulty, BlockInfo, BlockRanks},
         Blockchain, BlockchainOverlay,
     },
     error::TxVerifyFailed,
@@ -54,7 +54,7 @@ pub mod fees;
 
 /// Helper utilities
 pub mod utils;
-use utils::deploy_native_contracts;
+use utils::{block_rank, deploy_native_contracts};
 
 /// Configuration for initializing [`Validator`]
 #[derive(Clone)]
@@ -400,6 +400,11 @@ impl Validator {
         // Retrieve last block
         let mut previous = &overlay.lock().unwrap().last_block()?;
 
+        // Retrieve last block difficulty to access current ranks
+        let last_difficulty = self.blockchain.last_block_difficulty()?;
+        let mut current_targets_rank = last_difficulty.ranks.targets_rank;
+        let mut current_hashes_rank = last_difficulty.ranks.hashes_rank;
+
         // Grab current PoW module to validate each block
         let mut module = self.consensus.module.read().await.clone();
 
@@ -421,14 +426,31 @@ impl Validator {
                 return Err(Error::BlockIsInvalid(block.hash()?.to_string()))
             };
 
-            // Generate block difficulty
-            let difficulty = module.next_difficulty()?;
-            let cummulative_difficulty = module.cummulative_difficulty.clone() + difficulty.clone();
+            // Grab next mine target and difficulty
+            let (next_target, next_difficulty) = module.next_mine_target_and_difficulty()?;
+
+            // Calculate block rank
+            let (target_distance_sq, hash_distance_sq) = block_rank(block, &next_target)?;
+
+            // Update current ranks
+            current_targets_rank += target_distance_sq.clone();
+            current_hashes_rank += hash_distance_sq.clone();
+
+            // Generate block difficulty and update PoW module
+            let cummulative_difficulty =
+                module.cummulative_difficulty.clone() + next_difficulty.clone();
+            let ranks = BlockRanks::new(
+                target_distance_sq,
+                current_targets_rank.clone(),
+                hash_distance_sq,
+                current_hashes_rank.clone(),
+            );
             let block_difficulty = BlockDifficulty::new(
                 block.header.height,
                 block.header.timestamp,
-                difficulty,
+                next_difficulty,
                 cummulative_difficulty,
+                ranks,
             );
             module.append_difficulty(&overlay, block_difficulty)?;