blockchain/Block: removed slots vector, and handle storing based on block version.

With these changes, BlockInfo now represents a wrapper over Block, as it becomes constant/final in terms of structure. All extra data based on block versions go to BlockInfo, and their storage is handled accordingly.

src/blockchain/block_store.rs

@@ -36,9 +36,10 @@ pub const BLOCK_VERSION: u8 = 1;
 /// Block magic bytes
 const BLOCK_MAGIC_BYTES: [u8; 4] = [0x11, 0x6d, 0x75, 0x1f];
 
-/// This struct represents a tuple of the form (`magic`, `header`, `txs`, `producer`, `slots`).
-/// The header and transactions are stored as hashes, while slots are stored as integers,
-/// serving as pointers to the actual data in the sled database.
+/// This struct represents a tuple of the form (`magic`, `header`, `txs`, `signature`, `eta`).
+/// The header and transactions are stored as hashes, serving as pointers to the actual data
+/// in the sled database.
+/// NOTE: This struct fields are considered final, as it represents a blockchain block.
 #[derive(Debug, Clone, SerialEncodable, SerialDecodable)]
 pub struct Block {
     /// Block magic bytes
@@ -51,8 +52,6 @@ pub struct Block {
     pub signature: Signature,
     /// Block producer ETA
     pub eta: pallas::Base,
-    /// Slots up until this block
-    pub slots: Vec<u64>,
 }
 
 impl Block {
@@ -61,10 +60,9 @@ impl Block {
         txs: Vec<blake3::Hash>,
         signature: Signature,
         eta: pallas::Base,
-        slots: Vec<u64>,
     ) -> Self {
         let magic = BLOCK_MAGIC_BYTES;
-        Self { magic, header, txs, signature, eta, slots }
+        Self { magic, header, txs, signature, eta }
     }
 
     /// Calculate the block hash
@@ -73,7 +71,10 @@ impl Block {
     }
 }
 
-/// Structure representing full block data.
+/// Structure representing full block data, acting as
+/// a wrapper struct over `Block`, enabling us to include
+/// more information that might be used in different block
+/// version, without affecting the original struct.
 #[derive(Debug, Clone, SerialEncodable, SerialDecodable)]
 pub struct BlockInfo {
     /// Block magic bytes
@@ -127,14 +128,12 @@ impl BlockInfo {
 impl From<BlockInfo> for Block {
     fn from(block_info: BlockInfo) -> Self {
         let txs = block_info.txs.iter().map(|x| blake3::hash(&serialize(x))).collect();
-        let slots = block_info.slots.iter().map(|x| x.id).collect();
         Self {
             magic: block_info.magic,
             header: block_info.header.headerhash().unwrap(),
             txs,
             signature: block_info.signature,
             eta: block_info.eta,
-            slots,
         }
     }
 }

src/blockchain/mod.rs

@@ -38,7 +38,7 @@ pub use header_store::{Header, HeaderStore, HeaderStoreOverlay};
 
 /// Slots storage implementation
 pub mod slot_store;
-pub use slot_store::{SlotStore, SlotStoreOverlay};
+pub use slot_store::{BlocksSlotsStore, BlocksSlotsStoreOverlay, SlotStore, SlotStoreOverlay};
 
 /// Transactions related storage implementations
 pub mod tx_store;
@@ -63,6 +63,8 @@ pub struct Blockchain {
     pub order: BlockOrderStore,
     /// Slot sled tree
     pub slots: SlotStore,
+    /// Blocks Slots sled tree
+    pub blocks_slots: BlocksSlotsStore,
     /// Transactions sled tree
     pub transactions: TxStore,
     /// Pending transactions sled tree
@@ -82,6 +84,7 @@ impl Blockchain {
         let blocks = BlockStore::new(db)?;
         let order = BlockOrderStore::new(db)?;
         let slots = SlotStore::new(db)?;
+        let blocks_slots = BlocksSlotsStore::new(db)?;
         let transactions = TxStore::new(db)?;
         let pending_txs = PendingTxStore::new(db)?;
         let pending_txs_order = PendingTxOrderStore::new(db)?;
@@ -94,6 +97,7 @@ impl Blockchain {
             blocks,
             order,
             slots,
+            blocks_slots,
             transactions,
             pending_txs,
             pending_txs_order,
@@ -125,18 +129,28 @@ impl Blockchain {
         let blk: Block = Block::from(block.clone());
         let (bocks_batch, block_hashes) = self.blocks.insert_batch(&[blk])?;
         let block_hash = block_hashes[0];
+        let block_hash_vec = [block_hash];
         trees.push(self.blocks.0.clone());
         batches.push(bocks_batch);
 
         // Store block order
-        let blocks_order_batch = self.order.insert_batch(&[block.header.slot], &[block_hash])?;
+        let blocks_order_batch = self.order.insert_batch(&[block.header.slot], &block_hash_vec)?;
         trees.push(self.order.0.clone());
         batches.push(blocks_order_batch);
 
-        // Store slot checkpoints
-        let slots_batch = self.slots.insert_batch(&block.slots)?;
-        trees.push(self.slots.0.clone());
-        batches.push(slots_batch);
+        // Store extra stuff based on block version
+        if block.header.version > 0 {
+            // Store block slots uids vector
+            let slots = block.slots.iter().map(|x| x.id).collect();
+            let blocks_slots_bactch = self.blocks_slots.insert_batch(&block_hash_vec, &[&slots])?;
+            trees.push(self.blocks_slots.0.clone());
+            batches.push(blocks_slots_bactch);
+
+            // Store block slots
+            let slots_batch = self.slots.insert_batch(&block.slots)?;
+            trees.push(self.slots.0.clone());
+            batches.push(slots_batch);
+        }
 
         // Perform an atomic transaction over the trees and apply the batches.
         self.atomic_write(&trees, &batches)?;
@@ -158,10 +172,22 @@ impl Blockchain {
             return Ok(false)
         }
 
-        // Check if we have all slots
-        let slots: Vec<u64> = block.slots.iter().map(|x| x.id).collect();
-        if self.slots.get(&slots, true).is_err() {
-            return Ok(false)
+        // Check extra stuff based on block version
+        if block.header.version > 0 {
+            // Check if we have block slots uids vector
+            let slots = match self.blocks_slots.get(&[blockhash], true) {
+                Ok(v) => v[0].clone().unwrap(),
+                Err(_) => return Ok(false),
+            };
+            let provided_block_slots: Vec<u64> = block.slots.iter().map(|x| x.id).collect();
+            if slots != provided_block_slots {
+                return Ok(false)
+            }
+
+            // Check if we have all slots
+            if self.slots.get(&slots, true).is_err() {
+                return Ok(false)
+            }
         }
 
         // Check provided info produces the same hash
@@ -189,10 +215,16 @@ impl Blockchain {
             let txs = self.transactions.get(&block.txs, true)?;
             let txs = txs.iter().map(|x| x.clone().unwrap()).collect();
 
-            let slots = self.slots.get(&block.slots, true)?;
-            let slots = slots.iter().map(|x| x.clone().unwrap()).collect();
+            // Retrieve extra stuff based on block version
+            let mut block_slots = vec![];
+            if header.version > 0 {
+                let slots = self.blocks_slots.get(&[block.blockhash()], true)?;
+                let slots = slots[0].clone().unwrap();
+                let slots = self.slots.get(&slots, true)?;
+                block_slots = slots.iter().map(|x| x.clone().unwrap()).collect();
+            }
 
-            let info = BlockInfo::new(header, txs, block.signature, block.eta, slots);
+            let info = BlockInfo::new(header, txs, block.signature, block.eta, block_slots);
             ret.push(info);
         }
 
@@ -318,8 +350,8 @@ impl Blockchain {
         let txs_hashes: Vec<blake3::Hash> =
             txs.iter().map(|x| blake3::hash(&serialize(x))).collect();
         let indexes = self.pending_txs_order.get_all()?;
-        // We could do indexes.iter().map(|x| txs_hashes.contains(x.1)).collect.map(|x| x.0).collect but this is faster
-        // since we don't do the second iteration
+        // We could do indexes.iter().map(|x| txs_hashes.contains(x.1)).collect.map(|x| x.0).collect
+        // but this is faster since we don't do the second iteration
         let mut removed_indexes = vec![];
         for index in indexes {
             if txs_hashes.contains(&index.1) {
@@ -384,6 +416,8 @@ pub struct BlockchainOverlay {
     pub order: BlockOrderStoreOverlay,
     /// Slots overlay
     pub slots: SlotStoreOverlay,
+    /// Blocks slots overlay
+    pub blocks_slots: BlocksSlotsStoreOverlay,
     /// Transactions overlay
     pub transactions: TxStoreOverlay,
     /// Contract states overlay
@@ -400,6 +434,7 @@ impl BlockchainOverlay {
         let blocks = BlockStoreOverlay::new(&overlay)?;
         let order = BlockOrderStoreOverlay::new(&overlay)?;
         let slots = SlotStoreOverlay::new(&overlay)?;
+        let blocks_slots = BlocksSlotsStoreOverlay::new(&overlay)?;
         let transactions = TxStoreOverlay::new(&overlay)?;
         let contracts = ContractStateStoreOverlay::new(&overlay)?;
         let wasm_bincode = WasmStoreOverlay::new(&overlay)?;
@@ -410,6 +445,7 @@ impl BlockchainOverlay {
             blocks,
             order,
             slots,
+            blocks_slots,
             transactions,
             contracts,
             wasm_bincode,
@@ -449,12 +485,20 @@ impl BlockchainOverlay {
         // Store block
         let blk: Block = Block::from(block.clone());
         let block_hash = self.blocks.insert(&[blk])?[0];
+        let block_hash_vec = [block_hash];
 
         // Store block order
-        self.order.insert(&[block.header.slot], &[block_hash])?;
+        self.order.insert(&[block.header.slot], &block_hash_vec)?;
 
-        // Store slot checkpoints
-        self.slots.insert(&block.slots)?;
+        // Store extra stuff based on block version
+        if block.header.version > 0 {
+            // Store block slots uids vector
+            let slots = block.slots.iter().map(|x| x.id).collect();
+            self.blocks_slots.insert(&block_hash_vec, &[&slots])?;
+
+            // Store block slots
+            self.slots.insert(&block.slots)?;
+        }
 
         Ok(block_hash)
     }
@@ -473,10 +517,22 @@ impl BlockchainOverlay {
             return Ok(false)
         }
 
-        // Check if we have all slots
-        let slots: Vec<u64> = block.slots.iter().map(|x| x.id).collect();
-        if self.slots.get(&slots, true).is_err() {
-            return Ok(false)
+        // Check extra stuff based on block version
+        if block.header.version > 0 {
+            // Check if we have block slots uids vector
+            let slots = match self.blocks_slots.get(&[blockhash], true) {
+                Ok(v) => v[0].clone().unwrap(),
+                Err(_) => return Ok(false),
+            };
+            let provided_block_slots: Vec<u64> = block.slots.iter().map(|x| x.id).collect();
+            if slots != provided_block_slots {
+                return Ok(false)
+            }
+
+            // Check if we have all slots
+            if self.slots.get(&slots, true).is_err() {
+                return Ok(false)
+            }
         }
 
         // Check provided info produces the same hash
@@ -504,10 +560,16 @@ impl BlockchainOverlay {
             let txs = self.transactions.get(&block.txs, true)?;
             let txs = txs.iter().map(|x| x.clone().unwrap()).collect();
 
-            let slots = self.slots.get(&block.slots, true)?;
-            let slots = slots.iter().map(|x| x.clone().unwrap()).collect();
+            // Retrieve extra stuff based on block version
+            let mut block_slots = vec![];
+            if header.version > 0 {
+                let slots = self.blocks_slots.get(&[block.blockhash()], true)?;
+                let slots = slots[0].clone().unwrap();
+                let slots = self.slots.get(&slots, true)?;
+                block_slots = slots.iter().map(|x| x.clone().unwrap()).collect();
+            }
 
-            let info = BlockInfo::new(header, txs, block.signature, block.eta, slots);
+            let info = BlockInfo::new(header, txs, block.signature, block.eta, block_slots);
             ret.push(info);
         }
 
@@ -534,6 +596,7 @@ impl BlockchainOverlay {
         let blocks = BlockStoreOverlay::new(&overlay)?;
         let order = BlockOrderStoreOverlay::new(&overlay)?;
         let slots = SlotStoreOverlay::new(&overlay)?;
+        let blocks_slots = BlocksSlotsStoreOverlay::new(&overlay)?;
         let transactions = TxStoreOverlay::new(&overlay)?;
         let contracts = ContractStateStoreOverlay::new(&overlay)?;
         let wasm_bincode = WasmStoreOverlay::new(&overlay)?;
@@ -544,6 +607,7 @@ impl BlockchainOverlay {
             blocks,
             order,
             slots,
+            blocks_slots,
             transactions,
             contracts,
             wasm_bincode,

src/blockchain/slot_store.rs

@@ -22,7 +22,7 @@ use darkfi_serial::{deserialize, serialize};
 
 use crate::{Error, Result};
 
-use super::{parse_u64_key_record, SledDbOverlayPtr};
+use super::{parse_record, parse_u64_key_record, SledDbOverlayPtr};
 
 const SLED_SLOT_TREE: &[u8] = b"_slots";
 
@@ -203,3 +203,151 @@ impl SlotStoreOverlay {
         Ok(slot)
     }
 }
+
+const SLED_BLOCK_SLOTS_TREE: &[u8] = b"_blocks_slots";
+
+/// The `BlocksSlotsStore` is a `sled` tree storing all the blocks' corresponding slot
+/// uids, meaning the slot numbers leading up to each block, where the key is the
+/// blocks' hash, and value is the serialized slot uids vector.
+#[derive(Clone)]
+pub struct BlocksSlotsStore(pub sled::Tree);
+
+impl BlocksSlotsStore {
+    /// Opens a new or existing `BlocksSlotsStore` on the given sled database.
+    pub fn new(db: &sled::Db) -> Result<Self> {
+        let tree = db.open_tree(SLED_BLOCK_SLOTS_TREE)?;
+        Ok(Self(tree))
+    }
+
+    /// Insert a slice of block hashes and their `u64` vectors into the store.
+    pub fn insert(&self, hashes: &[blake3::Hash], slots: &[&Vec<u64>]) -> Result<()> {
+        let batch = self.insert_batch(hashes, slots)?;
+        self.0.apply_batch(batch)?;
+        Ok(())
+    }
+
+    /// Generate the sled batch corresponding to an insert, so caller
+    /// can handle the write operation. The block hash is used as the key,
+    /// and the block slots serialized vector is used as value.
+    pub fn insert_batch(
+        &self,
+        hashes: &[blake3::Hash],
+        slots: &[&Vec<u64>],
+    ) -> Result<sled::Batch> {
+        if hashes.len() != slots.len() {
+            return Err(Error::InvalidInputLengths)
+        }
+
+        let mut batch = sled::Batch::default();
+
+        for (i, hash) in hashes.iter().enumerate() {
+            let serialized = serialize(slots[i]);
+            batch.insert(hash.as_bytes(), serialized);
+        }
+
+        Ok(batch)
+    }
+
+    /// Check if the blocks slots store contains a given block hash.
+    pub fn contains(&self, blockhash: &blake3::Hash) -> Result<bool> {
+        Ok(self.0.contains_key(blockhash.as_bytes())?)
+    }
+
+    /// Fetch given blocks slots from the blocks slots store.
+    /// The resulting vector contains `Option`, which is `Some` if the block slots
+    /// were found in the blocks slots store, and otherwise it is `None`, if they have not.
+    /// The second parameter is a boolean which tells the function to fail in
+    /// case at least one block was not found.
+    pub fn get(
+        &self,
+        block_hashes: &[blake3::Hash],
+        strict: bool,
+    ) -> Result<Vec<Option<Vec<u64>>>> {
+        let mut ret = Vec::with_capacity(block_hashes.len());
+
+        for hash in block_hashes {
+            if let Some(found) = self.0.get(hash.as_bytes())? {
+                let slots = deserialize(&found)?;
+                ret.push(Some(slots));
+            } else {
+                if strict {
+                    let s = hash.to_hex().as_str().to_string();
+                    return Err(Error::BlockSlotsNotFound(s))
+                }
+                ret.push(None);
+            }
+        }
+
+        Ok(ret)
+    }
+
+    /// Retrieve all blocks slots from the block store in the form of a tuple
+    /// (`hash`, `Vec<u64>`).
+    /// Be careful as this will try to load everything in memory.
+    pub fn get_all(&self) -> Result<Vec<(blake3::Hash, Vec<u64>)>> {
+        let mut blocks_slots = vec![];
+
+        for block_slots in self.0.iter() {
+            blocks_slots.push(parse_record(block_slots.unwrap())?);
+        }
+
+        Ok(blocks_slots)
+    }
+}
+
+/// Overlay structure over a [`BlocksSlotsStore`] instance.
+pub struct BlocksSlotsStoreOverlay(SledDbOverlayPtr);
+
+impl BlocksSlotsStoreOverlay {
+    pub fn new(overlay: &SledDbOverlayPtr) -> Result<Self> {
+        overlay.lock().unwrap().open_tree(SLED_BLOCK_SLOTS_TREE)?;
+        Ok(Self(overlay.clone()))
+    }
+
+    /// Insert a slice of block hashes and their `u64` vectors into the overlay.
+    /// The block hash is used as the key, and the block slots serialized vector
+    /// is used as value.
+    pub fn insert(&self, hashes: &[blake3::Hash], slots: &[&Vec<u64>]) -> Result<()> {
+        if hashes.len() != slots.len() {
+            return Err(Error::InvalidInputLengths)
+        }
+
+        let mut lock = self.0.lock().unwrap();
+
+        for (i, hash) in hashes.iter().enumerate() {
+            let serialized = serialize(slots[i]);
+            lock.insert(SLED_BLOCK_SLOTS_TREE, hash.as_bytes(), &serialized)?;
+        }
+
+        Ok(())
+    }
+
+    /// Fetch given blocks slots from the overlay.
+    /// The resulting vector contains `Option`, which is `Some` if the block slots
+    /// were found in the overlay, and otherwise it is `None`, if they have not.
+    /// The second parameter is a boolean which tells the function to fail in
+    /// case at least one block was not found.
+    pub fn get(
+        &self,
+        block_hashes: &[blake3::Hash],
+        strict: bool,
+    ) -> Result<Vec<Option<Vec<u64>>>> {
+        let mut ret = Vec::with_capacity(block_hashes.len());
+        let lock = self.0.lock().unwrap();
+
+        for hash in block_hashes {
+            if let Some(found) = lock.get(SLED_BLOCK_SLOTS_TREE, hash.as_bytes())? {
+                let slots = deserialize(&found)?;
+                ret.push(Some(slots));
+            } else {
+                if strict {
+                    let s = hash.to_hex().as_str().to_string();
+                    return Err(Error::BlockSlotsNotFound(s))
+                }
+                ret.push(None);
+            }
+        }
+
+        Ok(ret)
+    }
+}

src/error.rs

@@ -372,6 +372,9 @@ pub enum Error {
     #[error("Slot {0} not found in database")]
     SlotNotFound(u64),
 
+    #[error("Block {0} slots not found in database")]
+    BlockSlotsNotFound(String),
+
     #[error("Future slot {0} was received")]
     FutureSlotReceived(u64),