src/consensu: block metadata modifications, proper hash handling, misc cleanups

script/research/nodes-tool/src/main.rs

@@ -4,7 +4,7 @@ use darkfi::{
     consensus::{
         block::{Block, BlockProposal, BlockStore},
         blockchain::{Blockchain, ProposalsChain},
-        metadata::{Metadata, MetadataStore, OuroborosMetadata, StreamletMetadata},
+        metadata::{Metadata, OuroborosMetadata, StreamletMetadata, StreamletMetadataStore},
         participant::Participant,
         state::{ConsensusState, ValidatorState},
         tx::{Tx, TxStore},
@@ -91,15 +91,13 @@ impl OuroborosMetadataInfo {
 struct MetadataInfo {
     _timestamp: Timestamp,
     _om: OuroborosMetadataInfo,
-    _sm: StreamletMetadataInfo,
 }
 
 impl MetadataInfo {
     pub fn new(metadata: &Metadata) -> MetadataInfo {
         let _timestamp = metadata.timestamp.clone();
         let _om = OuroborosMetadataInfo::new(&metadata.om);
-        let _sm = StreamletMetadataInfo::new(&metadata.sm);
-        MetadataInfo { _timestamp, _om, _sm }
+        MetadataInfo { _timestamp, _om }
     }
 }
 
@@ -110,6 +108,7 @@ struct ProposalInfo {
     _sl: u64,
     _txs: Vec<Tx>,
     _metadata: MetadataInfo,
+    _sm: StreamletMetadataInfo,
 }
 
 impl ProposalInfo {
@@ -119,7 +118,8 @@ impl ProposalInfo {
         let _sl = proposal.sl;
         let _txs = proposal.txs.clone();
         let _metadata = MetadataInfo::new(&proposal.metadata);
-        ProposalInfo { _id, _st, _sl, _txs, _metadata }
+        let _sm = StreamletMetadataInfo::new(&proposal.sm);
+        ProposalInfo { _id, _st, _sl, _txs, _metadata, _sm }
     }
 }
 
@@ -240,12 +240,12 @@ impl TxStoreInfo {
 #[derive(Debug)]
 struct HashedMetadataInfo {
     _block: blake3::Hash,
-    _metadata: MetadataInfo,
+    _metadata: StreamletMetadataInfo,
 }
 
 impl HashedMetadataInfo {
-    pub fn new(_block: blake3::Hash, metadata: &Metadata) -> HashedMetadataInfo {
-        let _metadata = MetadataInfo::new(&metadata);
+    pub fn new(_block: blake3::Hash, metadata: &StreamletMetadata) -> HashedMetadataInfo {
+        let _metadata = StreamletMetadataInfo::new(&metadata);
         HashedMetadataInfo { _block, _metadata }
     }
 }
@@ -256,7 +256,7 @@ struct MetadataStoreInfo {
 }
 
 impl MetadataStoreInfo {
-    pub fn new(metadatastore: &MetadataStore) -> MetadataStoreInfo {
+    pub fn new(metadatastore: &StreamletMetadataStore) -> MetadataStoreInfo {
         let mut _metadata = Vec::new();
         let result = metadatastore.get_all();
         match result {
@@ -287,7 +287,7 @@ impl BlockchainInfo {
     pub fn new(blockchain: &Blockchain) -> BlockchainInfo {
         let _blocks = BlockInfoChain::new(&blockchain.blocks);
         let _transactions = TxStoreInfo::new(&blockchain.transactions);
-        let _metadata = MetadataStoreInfo::new(&blockchain.metadata);
+        let _metadata = MetadataStoreInfo::new(&blockchain.streamlet_metadata);
         BlockchainInfo { _blocks, _transactions, _metadata }
     }
 }

src/consensus/block.rs

@@ -10,15 +10,14 @@ use crate::{
 };
 
 use super::{
-    metadata::Metadata,
-    participant::Participant,
+    metadata::{Metadata, StreamletMetadata},
     tx::Tx,
-    util::{Timestamp, GENESIS_HASH_BYTES},
+    util::{Timestamp, EMPTY_HASH_BYTES},
 };
 
 const SLED_BLOCK_TREE: &[u8] = b"_blocks";
 
-/// This struct represents a tuple of the form (st, sl, txs).
+/// This struct represents a tuple of the form (st, sl, txs, metadata).
 #[derive(Debug, Clone, SerialEncodable, SerialDecodable)]
 pub struct Block {
     /// Previous block hash
@@ -28,11 +27,25 @@ pub struct Block {
     /// Transaction hashes
     /// The actual transactions are in [`TxStore`]
     pub txs: Vec<blake3::Hash>,
+    /// Additional block information
+    pub metadata: Metadata,
 }
 
 impl Block {
-    pub fn new(st: blake3::Hash, sl: u64, txs: Vec<blake3::Hash>) -> Block {
-        Block { st, sl, txs }
+    pub fn new(st: blake3::Hash, sl: u64, txs: Vec<blake3::Hash>, metadata: Metadata) -> Block {
+        Block { st, sl, txs, metadata }
+    }
+
+    /// Generates the genesis block.
+    pub fn genesis_block(genesis: i64) -> Block {
+        let hash = blake3::Hash::from(EMPTY_HASH_BYTES);
+        let metadata = Metadata::new(
+            Timestamp(genesis),
+            String::from("proof"),
+            String::from("r"),
+            String::from("s"),
+        );
+        Block::new(hash, 0, vec![], metadata)
     }
 }
 
@@ -41,14 +54,12 @@ pub struct BlockStore(sled::Tree);
 
 impl BlockStore {
     /// Opens a new or existing blockstore tree given a sled database.
-    pub fn new(db: &sled::Db) -> Result<Self> {
+    pub fn new(db: &sled::Db, genesis: i64) -> Result<Self> {
         let tree = db.open_tree(SLED_BLOCK_TREE)?;
         let store = Self(tree);
         if store.0.is_empty() {
             // Genesis block is generated.
-            let hash = blake3::Hash::from(GENESIS_HASH_BYTES);
-            let genesis_block = Block::new(hash, 0, vec![]);
-            store.insert(&genesis_block)?;
+            store.insert(&Block::genesis_block(genesis))?;
         }
 
         Ok(store)
@@ -97,6 +108,8 @@ pub struct BlockProposal {
     pub txs: Vec<Tx>,
     /// Additional proposal information
     pub metadata: Metadata,
+    /// Proposal information used by Streamlet consensus
+    pub sm: StreamletMetadata,
 }
 
 impl BlockProposal {
@@ -107,21 +120,31 @@ impl BlockProposal {
         st: blake3::Hash,
         sl: u64,
         txs: Vec<Tx>,
-        timestamp: Timestamp,
-        proof: String,
-        r: String,
-        s: String,
-        participants: Vec<Participant>,
+        metadata: Metadata,
+        sm: StreamletMetadata,
     ) -> BlockProposal {
-        BlockProposal {
-            public_key,
-            signature,
-            id,
-            st,
-            sl,
-            txs,
-            metadata: Metadata::new(timestamp, proof, r, s, participants),
+        BlockProposal { public_key, signature, id, st, sl, txs, metadata, sm }
+    }
+
+    /// Produce proposal hash using st, sl, txs and metadata.
+    pub fn hash(&self) -> blake3::Hash {
+        Self::to_proposal_hash(self.st, self.sl, &self.txs, &self.metadata)
+    }
+
+    /// Util function generate a proposal hash using provided st, sl, txs and metadata.
+    pub fn to_proposal_hash(
+        st: blake3::Hash,
+        sl: u64,
+        transactions: &Vec<Tx>,
+        metadata: &Metadata,
+    ) -> blake3::Hash {
+        let mut txs = Vec::new();
+        for tx in transactions {
+            let hash = blake3::hash(&serialize(tx));
+            txs.push(hash);
         }
+
+        blake3::hash(&serialize(&Block::new(st, sl, txs, metadata.clone())))
     }
 }
 
@@ -132,7 +155,8 @@ impl PartialEq for BlockProposal {
             self.id == other.id &&
             self.st == other.st &&
             self.sl == other.sl &&
-            self.txs == other.txs
+            self.txs == other.txs &&
+            self.metadata == other.metadata
     }
 }
 

src/consensus/blockchain.rs

@@ -10,25 +10,27 @@ use crate::{
 
 use super::{
     block::{Block, BlockProposal, BlockStore},
-    metadata::MetadataStore,
+    metadata::StreamletMetadataStore,
     tx::TxStore,
-    util::{to_block_serial, GENESIS_HASH_BYTES},
 };
 
-/// This struct represents a sequence of blocks starting with the genesis block.
+/// This struct represents the canonical (finalized) blockchain stored in sled database.
 #[derive(Debug)]
 pub struct Blockchain {
+    /// Blocks sled database
     pub blocks: BlockStore,
+    /// Transactions sled database
     pub transactions: TxStore,
-    pub metadata: MetadataStore,
+    /// Streamlet metadata sled database
+    pub streamlet_metadata: StreamletMetadataStore,
 }
 
 impl Blockchain {
-    pub fn new(db: &sled::Db) -> Result<Blockchain> {
-        let blocks = BlockStore::new(db)?;
+    pub fn new(db: &sled::Db, genesis: i64) -> Result<Blockchain> {
+        let blocks = BlockStore::new(db, genesis)?;
         let transactions = TxStore::new(db)?;
-        let metadata = MetadataStore::new(db)?;
-        Ok(Blockchain { blocks, transactions, metadata })
+        let streamlet_metadata = StreamletMetadataStore::new(db)?;
+        Ok(Blockchain { blocks, transactions, streamlet_metadata })
     }
 
     /// Insertion of a block proposal.
@@ -41,11 +43,11 @@ impl Blockchain {
         }
 
         // Storing block
-        let block = Block { st: proposal.st, sl: proposal.sl, txs };
+        let block = Block { st: proposal.st, sl: proposal.sl, txs, metadata: proposal.metadata };
         let hash = self.blocks.insert(&block)?;
 
-        // Storing metadata
-        self.metadata.insert(&proposal.metadata, hash)?;
+        // Storing streamlet metadata
+        self.streamlet_metadata.insert(hash, &proposal.sm)?;
 
         Ok(hash)
     }
@@ -69,23 +71,24 @@ impl ProposalsChain {
         &self,
         proposal: &BlockProposal,
         previous: &BlockProposal,
-    ) -> Result<bool> {
-        if proposal.st.as_bytes() == &GENESIS_HASH_BYTES {
+        genesis: &blake3::Hash,
+    ) -> bool {
+        if &proposal.st == genesis {
             debug!("Genesis block proposal provided.");
-            return Ok(false)
+            return false
         }
-        let previous_hash = blake3::hash(&to_block_serial(previous.st, previous.sl, &previous.txs));
+        let previous_hash = previous.hash();
         if proposal.st != previous_hash || proposal.sl <= previous.sl {
             debug!("Provided proposal is invalid.");
-            return Ok(false)
+            return false
         }
-        Ok(true)
+        true
     }
 
     /// A proposals chain is considered valid, when every proposal is valid, based on check_proposal function.
-    pub fn check_chain(&self) -> bool {
+    pub fn check_chain(&self, genesis: &blake3::Hash) -> bool {
         for (index, proposal) in self.proposals[1..].iter().enumerate() {
-            if !self.check_proposal(proposal, &self.proposals[index]).unwrap() {
+            if !self.check_proposal(proposal, &self.proposals[index], genesis) {
                 return false
             }
         }
@@ -93,15 +96,16 @@ impl ProposalsChain {
     }
 
     /// Insertion of a valid proposal.
-    pub fn add(&mut self, proposal: &BlockProposal) {
-        self.check_proposal(proposal, self.proposals.last().unwrap()).unwrap();
-        self.proposals.push(proposal.clone());
+    pub fn add(&mut self, proposal: &BlockProposal, genesis: &blake3::Hash) {
+        if self.check_proposal(proposal, self.proposals.last().unwrap(), genesis) {
+            self.proposals.push(proposal.clone());
+        }
     }
 
     /// Proposals chain notarization check.
     pub fn notarized(&self) -> bool {
         for proposal in &self.proposals {
-            if !proposal.metadata.sm.notarized {
+            if !proposal.sm.notarized {
                 return false
             }
         }

src/consensus/metadata.rs

@@ -5,37 +5,25 @@ use crate::{
 
 use super::{participant::Participant, util::Timestamp, vote::Vote};
 
-const SLED_METADATA_TREE: &[u8] = b"_metadata";
+const SLED_STREAMLET_METADATA_TREE: &[u8] = b"_streamlet_metadata";
 
 /// This struct represents additional Block information used by the consensus protocol.
-#[derive(Debug, Clone, SerialEncodable, SerialDecodable)]
+#[derive(Debug, Clone, PartialEq, SerialEncodable, SerialDecodable)]
 pub struct Metadata {
     /// Block creation timestamp
     pub timestamp: Timestamp,
     /// Block information used by Ouroboros consensus
     pub om: OuroborosMetadata,
-    /// Block information used by Streamlet consensus
-    pub sm: StreamletMetadata,
 }
 
 impl Metadata {
-    pub fn new(
-        timestamp: Timestamp,
-        proof: String,
-        r: String,
-        s: String,
-        participants: Vec<Participant>,
-    ) -> Metadata {
-        Metadata {
-            timestamp,
-            om: OuroborosMetadata::new(proof, r, s),
-            sm: StreamletMetadata::new(participants),
-        }
+    pub fn new(timestamp: Timestamp, proof: String, r: String, s: String) -> Metadata {
+        Metadata { timestamp, om: OuroborosMetadata::new(proof, r, s) }
     }
 }
 
 /// This struct represents Block information used by Ouroboros consensus protocol.
-#[derive(Debug, Clone, SerialEncodable, SerialDecodable)]
+#[derive(Debug, Clone, PartialEq, SerialEncodable, SerialDecodable)]
 pub struct OuroborosMetadata {
     /// Proof the stakeholder is the block owner
     pub proof: String,
@@ -71,24 +59,24 @@ impl StreamletMetadata {
 }
 
 #[derive(Debug)]
-pub struct MetadataStore(sled::Tree);
+pub struct StreamletMetadataStore(sled::Tree);
 
-impl MetadataStore {
+impl StreamletMetadataStore {
     pub fn new(db: &sled::Db) -> Result<Self> {
-        let tree = db.open_tree(SLED_METADATA_TREE)?;
+        let tree = db.open_tree(SLED_STREAMLET_METADATA_TREE)?;
         Ok(Self(tree))
     }
 
-    /// Insert metadata into the metadatastore.
+    /// Insert streamlet metadata into the store.
     /// The block hash for the metadata is used as the key, where value is the serialized metadata.
-    pub fn insert(&self, metadata: &Metadata, block: blake3::Hash) -> Result<()> {
+    pub fn insert(&self, block: blake3::Hash, metadata: &StreamletMetadata) -> Result<()> {
         self.0.insert(block.as_bytes(), serialize(metadata))?;
         Ok(())
     }
 
-    /// Retrieve all metadata.
+    /// Retrieve all streamlet metadata.
     /// Be carefull as this will try to load everything in memory.
-    pub fn get_all(&self) -> Result<Vec<Option<(blake3::Hash, Metadata)>>> {
+    pub fn get_all(&self) -> Result<Vec<Option<(blake3::Hash, StreamletMetadata)>>> {
         let mut metadata = Vec::new();
         let mut iterator = self.0.into_iter().enumerate();
         while let Some((_, r)) = iterator.next() {

src/consensus/state.rs

@@ -19,11 +19,12 @@ use crate::{
 };
 
 use super::{
-    block::BlockProposal,
+    block::{Block, BlockProposal},
     blockchain::{Blockchain, ProposalsChain},
+    metadata::{Metadata, StreamletMetadata},
     participant::Participant,
     tx::Tx,
-    util::{get_current_time, to_block_serial, Timestamp, GENESIS_HASH_BYTES},
+    util::{get_current_time, Timestamp},
     vote::Vote,
 };
 
@@ -56,11 +57,9 @@ impl ConsensusState {
         let consensus = if let Some(found) = tree.get(id.to_ne_bytes())? {
             deserialize(&found).unwrap()
         } else {
-            let hash = blake3::Hash::from(GENESIS_HASH_BYTES);
-            let genesis_hash = blake3::hash(&to_block_serial(hash, 0, &vec![]));
             let consensus = ConsensusState {
                 genesis: Timestamp(genesis),
-                last_block: genesis_hash,
+                last_block: blake3::hash(&serialize(&Block::genesis_block(genesis))),
                 last_sl: 0,
                 proposals: Vec::new(),
                 orphan_votes: Vec::new(),
@@ -94,6 +93,8 @@ pub struct ValidatorState {
     pub blockchain: Blockchain,
     /// Pending transactions
     pub unconfirmed_txs: Vec<Tx>,
+    /// Genesis block hash, used for validations
+    pub genesis_block: blake3::Hash,
 }
 
 impl ValidatorState {
@@ -103,8 +104,9 @@ impl ValidatorState {
         let db = sled::open(db_path)?;
         let public = PublicKey::from_secret(secret);
         let consensus = ConsensusState::new(&db, id, genesis)?;
-        let blockchain = Blockchain::new(&db)?;
+        let blockchain = Blockchain::new(&db, genesis)?;
         let unconfirmed_txs = Vec::new();
+        let genesis_block = blake3::hash(&serialize(&Block::genesis_block(genesis)));
         Ok(Arc::new(RwLock::new(ValidatorState {
             id,
             secret,
@@ -113,6 +115,7 @@ impl ValidatorState {
             consensus,
             blockchain,
             unconfirmed_txs,
+            genesis_block,
         })))
     }
 
@@ -170,8 +173,17 @@ impl ValidatorState {
         let epoch = self.current_epoch();
         let previous_hash = self.longest_notarized_chain_last_hash().unwrap();
         let unproposed_txs = self.unproposed_txs();
-        let signed_block =
-            self.secret.sign(&to_block_serial(previous_hash, epoch, &unproposed_txs)[..]);
+        let metadata = Metadata::new(
+            get_current_time(),
+            String::from("proof"),
+            String::from("r"),
+            String::from("s"),
+        );
+        let sm = StreamletMetadata::new(self.consensus.participants.values().cloned().collect());
+        let signed_block = self.secret.sign(
+            BlockProposal::to_proposal_hash(previous_hash, epoch, &unproposed_txs, &metadata)
+                .as_bytes(),
+        );
         Ok(Some(BlockProposal::new(
             self.public,
             signed_block,
@@ -179,11 +191,8 @@ impl ValidatorState {
             previous_hash,
             epoch,
             unproposed_txs,
-            get_current_time(),
-            String::from("proof"),
-            String::from("r"),
-            String::from("s"),
-            self.consensus.participants.values().cloned().collect(),
+            metadata,
+            sm,
         )))
     }
 
@@ -215,8 +224,7 @@ impl ValidatorState {
                     }
                 }
             }
-            let last = longest_notarized_chain.proposals.last().unwrap();
-            blake3::hash(&to_block_serial(last.st, last.sl, &last.txs))
+            longest_notarized_chain.proposals.last().unwrap().hash()
         } else {
             self.consensus.last_block
         };
@@ -235,7 +243,13 @@ impl ValidatorState {
             return Ok(None)
         }
         if !proposal.public_key.verify(
-            &to_block_serial(proposal.st, proposal.sl, &proposal.txs)[..],
+            BlockProposal::to_proposal_hash(
+                proposal.st,
+                proposal.sl,
+                &proposal.txs,
+                &proposal.metadata,
+            )
+            .as_bytes(),
             &proposal.signature,
         ) {
             debug!("Proposer signature couldn't be verified. Proposer: {:?}", proposal.id);
@@ -252,13 +266,13 @@ impl ValidatorState {
         let mut proposal = proposal.clone();
 
         // Generate proposal hash
-        let proposal_hash = blake3::hash(&to_block_serial(proposal.st, proposal.sl, &proposal.txs));
+        let proposal_hash = proposal.hash();
 
         // Add orphan votes
         let mut orphans = Vec::new();
         for vote in self.consensus.orphan_votes.iter() {
             if vote.proposal == proposal_hash {
-                proposal.metadata.sm.votes.push(vote.clone());
+                proposal.sm.votes.push(vote.clone());
                 orphans.push(vote.clone());
             }
         }
@@ -278,7 +292,7 @@ impl ValidatorState {
                 self.consensus.proposals.last().unwrap()
             }
             _ => {
-                self.consensus.proposals[index as usize].add(&proposal);
+                self.consensus.proposals[index as usize].add(&proposal, &self.genesis_block);
                 &self.consensus.proposals[index as usize]
             }
         };
@@ -299,7 +313,7 @@ impl ValidatorState {
     /// Node verifies if provided chain is notarized excluding the last block.
     pub fn extends_notarized_chain(&self, chain: &ProposalsChain) -> bool {
         for proposal in &chain.proposals[..(chain.proposals.len() - 1)] {
-            if !proposal.metadata.sm.notarized {
+            if !proposal.sm.notarized {
                 return false
             }
         }
@@ -310,7 +324,7 @@ impl ValidatorState {
     pub fn find_extended_chain_index(&self, proposal: &BlockProposal) -> Result<i64> {
         for (index, chain) in self.consensus.proposals.iter().enumerate() {
             let last = chain.proposals.last().unwrap();
-            let hash = blake3::hash(&to_block_serial(last.st, last.sl, &last.txs));
+            let hash = last.hash();
             if proposal.st == hash && proposal.sl > last.sl {
                 return Ok(index as i64)
             }
@@ -380,13 +394,11 @@ impl ValidatorState {
         }
 
         let (unwrapped, chain_index) = proposal.unwrap();
-        if !unwrapped.metadata.sm.votes.contains(vote) {
-            unwrapped.metadata.sm.votes.push(vote.clone());
+        if !unwrapped.sm.votes.contains(vote) {
+            unwrapped.sm.votes.push(vote.clone());
 
-            if !unwrapped.metadata.sm.notarized &&
-                unwrapped.metadata.sm.votes.len() > (2 * nodes_count / 3)
-            {
-                unwrapped.metadata.sm.notarized = true;
+            if !unwrapped.sm.notarized && unwrapped.sm.votes.len() > (2 * nodes_count / 3) {
+                unwrapped.sm.notarized = true;
                 self.chain_finalization(chain_index)?;
             }
 
@@ -420,8 +432,7 @@ impl ValidatorState {
     ) -> Result<Option<(&mut BlockProposal, i64)>> {
         for (index, chain) in &mut self.consensus.proposals.iter_mut().enumerate() {
             for proposal in chain.proposals.iter_mut().rev() {
-                let proposal_hash =
-                    blake3::hash(&to_block_serial(proposal.st, proposal.sl, &proposal.txs));
+                let proposal_hash = proposal.hash();
                 if vote_proposal == &proposal_hash {
                     return Ok(Some((proposal, index as i64)))
                 }
@@ -440,7 +451,7 @@ impl ValidatorState {
         if len > 2 {
             let mut consecutive = 0;
             for proposal in &chain.proposals {
-                if proposal.metadata.sm.notarized {
+                if proposal.sm.notarized {
                     consecutive += 1;
                 } else {
                     break
@@ -450,7 +461,7 @@ impl ValidatorState {
             if consecutive > 2 {
                 let mut finalized = Vec::new();
                 for proposal in &mut chain.proposals[..(consecutive - 1)] {
-                    proposal.metadata.sm.finalized = true;
+                    proposal.sm.finalized = true;
                     finalized.push(proposal.clone());
                     for tx in proposal.txs.clone() {
                         if let Some(pos) = self.unconfirmed_txs.iter().position(|txs| *txs == tx) {

src/consensus/util.rs

@@ -1,11 +1,9 @@
 use chrono::{NaiveDateTime, Utc};
 
-use crate::util::serial::{serialize, SerialDecodable, SerialEncodable};
-
-use super::{block::Block, tx::Tx};
+use crate::util::serial::{SerialDecodable, SerialEncodable};
 
 /// Serialized blake3 hash bytes for character "⊥"
-pub const GENESIS_HASH_BYTES: [u8; 32] = [
+pub const EMPTY_HASH_BYTES: [u8; 32] = [
     254, 233, 82, 102, 23, 208, 153, 87, 96, 165, 163, 194, 238, 7, 1, 88, 14, 1, 249, 118, 197,
     29, 180, 211, 87, 66, 59, 38, 86, 54, 12, 39,
 ];
@@ -28,13 +26,3 @@ impl Timestamp {
 pub fn get_current_time() -> Timestamp {
     Timestamp(Utc::now().timestamp())
 }
-
-/// Util function to create a dummy block and encode it, to produce the correct hash
-pub fn to_block_serial(st: blake3::Hash, sl: u64, transactions: &Vec<Tx>) -> Vec<u8> {
-    let mut txs = Vec::new();
-    for tx in transactions {
-        let hash = blake3::hash(&serialize(tx));
-        txs.push(hash);
-    }
-    serialize(&Block::new(st, sl, txs))
-}