consesus:block restructure

doc/src/architecture/blockchain.md

@@ -224,26 +224,40 @@ without the need of forking the blockchain.
 | `blocks` | `Vec<Block>` | Series of blocks consisting the Blockchain |
 
 
+## Header
+
+|   Field     |        Type        |            Description                     |
+|-------------|--------------------|--------------------------------------------|
+| `v`         | `u8`               | Version                                    |
+| `st`        | `blake3Hash`       | Previous block hash                        |
+| `e`         | `u64`              | Epoch                                      |
+| `sl`        | `u64`              | Slot UID                                   |
+| `time`      | `Timestamp`        | Block creation timestamp                   |
+| `root`      | `MerkleRoot`       | Root of the transaction hashes merkle tree |
+
+
 ## Block
 
-|   Field    |        Type        |            Description            |
-|------------|--------------------|-----------------------------------|
-| `st`       | `String`           | Previous block hash               |
-| `sl`       | `u64`              | Slot UID, generated by the beacon |
-| `txs`      | `Vec<Transaction>` | Transactions payload              |
-| `metadata` | `Metadata`         | Additional block information      |
+|   Field    |        Type       |            Description             |
+|------------|-------------------|------------------------------------|
+| `magic`    | `u8`              | Magic bytes                        |
+| `header`   | `blake3Hash`      | Header hash                        |
+| `txs`      | `Vec<blake3Hash>` | Transaction hashes                 |
+| `metadata` | `Metadata`        | Additional block information       |
 
 
-## Metadata
+## BlockInfo
 
-|    Field    |         Type        |                  Description                  |
-|-------------|---------------------|-----------------------------------------------|
-| `om`        | `OuroborosMetadata` | Block information used by Ouroboros consensus |
-| `sm`        | `StreamletMetadata` | Block information used by Streamlet consensus |
-| `timestamp` | `Timestamp`         | Block creation timestamp                      |
+|   Field    |        Type         |            Description                           |
+|------------|---------------------|--------------------------------------------------|
+| `magic`    | `u8`                | Magic bytes                                      |
+| `header`   | `Header`            | Header data                                      |
+| `txs`      | `Vec<Transaction>`  | Transaction payload                              |
+| `metadata` | `Metadata`          | Additional block information                     |
+| `sm`       | `StreamletMetadata` | Proposal information used by Streamlet consensus |
 
 
-## Ouroboros Metadata
+## Metadata
 
 |    Field    |         Type        |                  Description                  |
 |-------------|---------------------|-----------------------------------------------|

script/research/nodes-tool/.gitignore

@@ -1,3 +1,5 @@
 /target
 Cargo.lock
 node*
+darkfid*
+faucetd*

script/research/nodes-tool/src/main.rs

@@ -3,23 +3,23 @@ use std::{fs::File, io::Write};
 
 use darkfi::{
     blockchain::{
-        blockstore::{BlockOrderStore, BlockStore},
+        blockstore::{BlockOrderStore, BlockStore, HeaderStore},
         metadatastore::StreamletMetadataStore,
         txstore::TxStore,
         Blockchain,
     },
     consensus::{
-        block::{Block, BlockProposal, ProposalChain},
-        metadata::{Metadata, OuroborosMetadata, StreamletMetadata},
+        block::{Block, BlockProposal, Header, ProposalChain},
+        metadata::{Metadata, StreamletMetadata},
         participant::Participant,
         state::{ConsensusState, ValidatorState},
         vote::Vote,
         TESTNET_GENESIS_HASH_BYTES,
     },
-    crypto::token_list::DrkTokenList,
+    crypto::{merkle_node::MerkleNode, token_list::DrkTokenList},
     node::Client,
     tx::Transaction,
-    util::{expand_path, time::Timestamp},
+    util::{expand_path, serial::serialize, time::Timestamp},
     wallet::walletdb::init_wallet,
     Result,
 };
@@ -81,54 +81,42 @@ impl StreamletMetadataInfo {
 }
 
 #[derive(Debug)]
-struct OuroborosMetadataInfo {
+struct MetadataInfo {
     _proof: String,
     _r: String,
     _s: String,
 }
 
-impl OuroborosMetadataInfo {
-    pub fn new(metadata: &OuroborosMetadata) -> OuroborosMetadataInfo {
+impl MetadataInfo {
+    pub fn new(metadata: &Metadata) -> MetadataInfo {
         let _proof = metadata.proof.clone();
         let _r = metadata.r.clone();
         let _s = metadata.s.clone();
-        OuroborosMetadataInfo { _proof, _r, _s }
-    }
-}
-
-#[derive(Debug)]
-struct MetadataInfo {
-    _timestamp: Timestamp,
-    _om: OuroborosMetadataInfo,
-}
-
-impl MetadataInfo {
-    pub fn new(metadata: &Metadata) -> MetadataInfo {
-        let _timestamp = metadata.timestamp.clone();
-        let _om = OuroborosMetadataInfo::new(&metadata.om);
-        MetadataInfo { _timestamp, _om }
+        MetadataInfo { _proof, _r, _s }
     }
 }
 
 #[derive(Debug)]
 struct ProposalInfo {
     _address: String,
-    _st: blake3::Hash,
-    _sl: u64,
-    _txs: Vec<Transaction>,
-    _metadata: MetadataInfo,
+    _block: BlockInfo,
     _sm: StreamletMetadataInfo,
 }
 
 impl ProposalInfo {
     pub fn new(proposal: &BlockProposal) -> ProposalInfo {
         let _address = proposal.address.to_string();
-        let _st = proposal.block.st;
-        let _sl = proposal.block.sl;
-        let _txs = proposal.block.txs.clone();
+        let _header = proposal.block.header.headerhash();
+        let mut _txs = vec![];
+        for tx in &proposal.block.txs {
+            let hash = blake3::hash(&serialize(tx));
+            _txs.push(hash);
+        }
         let _metadata = MetadataInfo::new(&proposal.block.metadata);
+        let _block =
+            BlockInfo { _hash: _header, _magic: proposal.block.magic, _header, _txs, _metadata };
         let _sm = StreamletMetadataInfo::new(&proposal.block.sm);
-        ProposalInfo { _address, _st, _sl, _txs, _metadata, _sm }
+        ProposalInfo { _address, _block, _sm }
     }
 }
 
@@ -165,19 +153,65 @@ impl ConsensusInfo {
 }
 
 #[derive(Debug)]
-struct BlockInfo {
+struct HeaderInfo {
     _hash: blake3::Hash,
+    _v: u8,
     _st: blake3::Hash,
+    _e: u64,
     _sl: u64,
+    _timestamp: Timestamp,
+    _root: MerkleNode,
+}
+
+impl HeaderInfo {
+    pub fn new(_hash: blake3::Hash, header: &Header) -> HeaderInfo {
+        let _v = header.v;
+        let _st = header.st;
+        let _e = header.e;
+        let _sl = header.sl;
+        let _timestamp = header.timestamp;
+        let _root = header.root;
+        HeaderInfo { _hash, _v, _st, _e, _sl, _timestamp, _root }
+    }
+}
+
+#[derive(Debug)]
+struct HeaderStoreInfo {
+    _headers: Vec<HeaderInfo>,
+}
+
+impl HeaderStoreInfo {
+    pub fn new(headerstore: &HeaderStore) -> HeaderStoreInfo {
+        let mut _headers = Vec::new();
+        let result = headerstore.get_all();
+        match result {
+            Ok(iter) => {
+                for (hash, header) in iter.iter() {
+                    _headers.push(HeaderInfo::new(hash.clone(), &header));
+                }
+            }
+            Err(e) => println!("Error: {:?}", e),
+        }
+        HeaderStoreInfo { _headers }
+    }
+}
+
+#[derive(Debug)]
+struct BlockInfo {
+    _hash: blake3::Hash,
+    _magic: [u8; 4],
+    _header: blake3::Hash,
     _txs: Vec<blake3::Hash>,
+    _metadata: MetadataInfo,
 }
 
 impl BlockInfo {
     pub fn new(_hash: blake3::Hash, block: &Block) -> BlockInfo {
-        let _st = block.st;
-        let _sl = block.sl;
+        let _magic = block.magic;
+        let _header = block.header;
         let _txs = block.txs.clone();
-        BlockInfo { _hash, _st, _sl, _txs }
+        let _metadata = MetadataInfo::new(&block.metadata);
+        BlockInfo { _hash, _magic, _header, _txs, _metadata }
     }
 }
 
@@ -305,6 +339,7 @@ impl MetadataStoreInfo {
 
 #[derive(Debug)]
 struct BlockchainInfo {
+    _headers: HeaderStoreInfo,
     _blocks: BlockInfoChain,
     _order: BlockOrderStoreInfo,
     _transactions: TxStoreInfo,
@@ -313,11 +348,12 @@ struct BlockchainInfo {
 
 impl BlockchainInfo {
     pub fn new(blockchain: &Blockchain) -> BlockchainInfo {
+        let _headers = HeaderStoreInfo::new(&blockchain.headers);
         let _blocks = BlockInfoChain::new(&blockchain.blocks);
         let _order = BlockOrderStoreInfo::new(&blockchain.order);
         let _transactions = TxStoreInfo::new(&blockchain.transactions);
         let _metadata = MetadataStoreInfo::new(&blockchain.streamlet_metadata);
-        BlockchainInfo { _blocks, _order, _transactions, _metadata }
+        BlockchainInfo { _headers, _blocks, _order, _transactions, _metadata }
     }
 }
 
@@ -337,41 +373,51 @@ impl StateInfo {
     }
 }
 
-#[async_std::main]
-async fn main() -> Result<()> {
-    let nodes = 4;
+async fn generate(name: &str, folder: &str) -> Result<()> {
     let genesis_ts = Timestamp(1648383795);
     let genesis_data = *TESTNET_GENESIS_HASH_BYTES;
     let pass = "changeme";
-    for i in 0..nodes {
-        // Initialize or load wallet
-        let path = format!("../../../tmp/node{:?}/wallet.db", i);
-        let wallet = init_wallet(&path, &pass).await?;
-        let address = wallet.get_default_address().await?;
-        let tokenlist = Arc::new(DrkTokenList::new(&[
-            ("drk", include_bytes!("../../../../contrib/token/darkfi_token_list.min.json")),
-            ("btc", include_bytes!("../../../../contrib/token/bitcoin_token_list.min.json")),
-            ("eth", include_bytes!("../../../../contrib/token/erc20_token_list.min.json")),
-            ("sol", include_bytes!("../../../../contrib/token/solana_token_list.min.json")),
-        ])?);
-        let client = Arc::new(Client::new(wallet, tokenlist).await?);
-
-        // Initialize or load sled database
-        let path = format!("../../../tmp/node{:?}/blockchain/testnet", i);
-        let db_path = expand_path(&path).unwrap();
-        let sled_db = sled::open(&db_path)?;
-
-        // Data export
-        println!("Exporting data for node{:?} - {:?}", i, address.to_string());
-        let state =
-            ValidatorState::new(&sled_db, genesis_ts, genesis_data, client, vec![], vec![]).await?;
-        let info = StateInfo::new(&*state.read().await);
-        let info_string = format!("{:#?}", info);
-        let path = format!("node{:?}_testnet_db", i);
-        let mut file = File::create(path)?;
-        file.write(info_string.as_bytes())?;
-        drop(sled_db);
-    }
+    // Initialize or load wallet
+    let path = folder.to_owned() + "/wallet.db";
+    let wallet = init_wallet(&path, &pass).await?;
+    let address = wallet.get_default_address().await?;
+    let tokenlist = Arc::new(DrkTokenList::new(&[
+        ("drk", include_bytes!("../../../../contrib/token/darkfi_token_list.min.json")),
+        ("btc", include_bytes!("../../../../contrib/token/bitcoin_token_list.min.json")),
+        ("eth", include_bytes!("../../../../contrib/token/erc20_token_list.min.json")),
+        ("sol", include_bytes!("../../../../contrib/token/solana_token_list.min.json")),
+    ])?);
+    let client = Arc::new(Client::new(wallet, tokenlist).await?);
+
+    // Initialize or load sled database
+    let path = folder.to_owned() + "/blockchain/testnet";
+    let db_path = expand_path(&path).unwrap();
+    let sled_db = sled::open(&db_path)?;
+
+    // Data export
+    println!("Exporting data for {:?} - {:?}", name, address.to_string());
+    let state =
+        ValidatorState::new(&sled_db, genesis_ts, genesis_data, client, vec![], vec![]).await?;
+    let info = StateInfo::new(&*state.read().await);
+    let info_string = format!("{:#?}", info);
+    let path = name.to_owned() + "_testnet_db";
+    let mut file = File::create(path)?;
+    file.write(info_string.as_bytes())?;
+    drop(sled_db);
+
+    Ok(())
+}
+
+#[async_std::main]
+async fn main() -> Result<()> {
+    // darkfid0
+    generate("darkfid0", "../../../contrib/localnet/darkfid0").await?;
+    // darkfid1
+    generate("darkfid1", "../../../contrib/localnet/darkfid1").await?;
+    // darkfid2
+    generate("darkfid2", "../../../contrib/localnet/darkfid2").await?;
+    // faucetd
+    generate("faucetd", "../../../contrib/localnet/faucetd").await?;
 
     Ok(())
 }

src/blockchain/blockstore.rs

@@ -1,5 +1,5 @@
 use crate::{
-    consensus::Block,
+    consensus::{Block, Header},
     util::{
         serial::{deserialize, serialize},
         time::Timestamp,
@@ -7,11 +7,98 @@ use crate::{
     Error, Result,
 };
 
+const SLED_HEADER_TREE: &[u8] = b"_headers";
 const SLED_BLOCK_TREE: &[u8] = b"_blocks";
 const SLED_BLOCK_ORDER_TREE: &[u8] = b"_block_order";
 
+/// The `HeaderStore` is a `sled` tree storing all the blockchain's blocks' headers
+/// where the key is the headers's hash, and value is the serialized header.
+#[derive(Clone)]
+pub struct HeaderStore(sled::Tree);
+
+impl HeaderStore {
+    /// Opens a new or existing `HeaderStore` on the given sled database.
+    pub fn new(db: &sled::Db, genesis_ts: Timestamp, genesis_data: blake3::Hash) -> Result<Self> {
+        let tree = db.open_tree(SLED_HEADER_TREE)?;
+        let store = Self(tree);
+
+        // In case the store is empty, initialize it with the genesis header.
+        if store.0.is_empty() {
+            let genesis_header = Header::genesis_header(genesis_ts, genesis_data);
+            store.insert(&[genesis_header])?;
+        }
+
+        Ok(store)
+    }
+
+    /// Insert a slice of [`Header`] into the blockstore. With sled, the
+    /// operation is done as a batch.
+    /// The headers are hashed with BLAKE3 and this headerhash is used as
+    /// the key, while value is the serialized [`Header`] itself.
+    /// On success, the function returns the header hashes in the same order.
+    pub fn insert(&self, headers: &[Header]) -> Result<Vec<blake3::Hash>> {
+        let mut ret = Vec::with_capacity(headers.len());
+        let mut batch = sled::Batch::default();
+
+        for header in headers {
+            let serialized = serialize(header);
+            let headerhash = blake3::hash(&serialized);
+            batch.insert(headerhash.as_bytes(), serialized);
+            ret.push(headerhash);
+        }
+
+        self.0.apply_batch(batch)?;
+        Ok(ret)
+    }
+
+    /// Check if the headerstore contains a given headerhash.
+    pub fn contains(&self, headerhash: &blake3::Hash) -> Result<bool> {
+        Ok(self.0.contains_key(headerhash.as_bytes())?)
+    }
+
+    /// Fetch given headerhashes from the headerstore.
+    /// The resulting vector contains `Option`, which is `Some` if the header
+    /// was found in the headerstore, and otherwise it is `None`, if it has not.
+    /// The second parameter is a boolean which tells the function to fail in
+    /// case at least one header was not found.
+    pub fn get(&self, headerhashes: &[blake3::Hash], strict: bool) -> Result<Vec<Option<Header>>> {
+        let mut ret = Vec::with_capacity(headerhashes.len());
+
+        for hash in headerhashes {
+            if let Some(found) = self.0.get(hash.as_bytes())? {
+                let header = deserialize(&found)?;
+                ret.push(Some(header));
+            } else {
+                if strict {
+                    let s = hash.to_hex().as_str().to_string();
+                    return Err(Error::HeaderNotFound(s))
+                }
+                ret.push(None);
+            }
+        }
+
+        Ok(ret)
+    }
+
+    /// Retrieve all headers from the headerstore in the form of a tuple
+    /// (`headerhash`, `header`).
+    /// Be careful as this will try to load everything in memory.
+    pub fn get_all(&self) -> Result<Vec<(blake3::Hash, Header)>> {
+        let mut headers = vec![];
+
+        for header in self.0.iter() {
+            let (key, value) = header.unwrap();
+            let hash_bytes: [u8; 32] = key.as_ref().try_into().unwrap();
+            let header = deserialize(&value)?;
+            headers.push((hash_bytes.into(), header));
+        }
+
+        Ok(headers)
+    }
+}
+
 /// The `BlockStore` is a `sled` tree storing all the blockchain's blocks
-/// where the key is the block's hash, and value is the serialized block.
+/// where the key is the block's headers' hash, and value is the serialized block.
 #[derive(Clone)]
 pub struct BlockStore(sled::Tree);
 
@@ -30,40 +117,34 @@ impl BlockStore {
         Ok(store)
     }
 
-    /// Insert a slice of [`Block`] into the blockstore. With sled, the
+    /// Insert a slice of [`Block`] into the store. With sled, the
     /// operation is done as a batch.
-    /// The blocks are hashed with BLAKE3 and this blockhash is used as
-    /// the key, while value is the serialized [`Block`] itself.
-    /// On success, the function returns the block hashes in the same order.
-    pub fn insert(&self, blocks: &[Block]) -> Result<Vec<blake3::Hash>> {
-        let mut ret = Vec::with_capacity(blocks.len());
+    /// The block's header is used as the key, while value is the serialized [`Block`] itself.
+    pub fn insert(&self, blocks: &[Block]) -> Result<()> {
         let mut batch = sled::Batch::default();
 
         for block in blocks {
-            let serialized = serialize(block);
-            let blockhash = blake3::hash(&serialized);
-            batch.insert(blockhash.as_bytes(), serialized);
-            ret.push(blockhash);
+            batch.insert(block.header.as_bytes(), serialize(block));
         }
 
         self.0.apply_batch(batch)?;
-        Ok(ret)
+        Ok(())
     }
 
-    /// Check if the blockstore contains a given blockhash.
-    pub fn contains(&self, blockhash: &blake3::Hash) -> Result<bool> {
-        Ok(self.0.contains_key(blockhash.as_bytes())?)
+    /// Check if the blockstore contains a given headerhash.
+    pub fn contains(&self, headerhash: &blake3::Hash) -> Result<bool> {
+        Ok(self.0.contains_key(headerhash.as_bytes())?)
     }
 
-    /// Fetch given blockhashes from the blockstore.
+    /// Fetch given headerhashes from the blockstore.
     /// The resulting vector contains `Option`, which is `Some` if the block
     /// was found in the blockstore, and otherwise it is `None`, if it has not.
     /// The second parameter is a boolean which tells the function to fail in
     /// case at least one block was not found.
-    pub fn get(&self, blockhashes: &[blake3::Hash], strict: bool) -> Result<Vec<Option<Block>>> {
-        let mut ret = Vec::with_capacity(blockhashes.len());
+    pub fn get(&self, headerhashes: &[blake3::Hash], strict: bool) -> Result<Vec<Option<Block>>> {
+        let mut ret = Vec::with_capacity(headerhashes.len());
 
-        for hash in blockhashes {
+        for hash in headerhashes {
             if let Some(found) = self.0.get(hash.as_bytes())? {
                 let block = deserialize(&found)?;
                 ret.push(Some(block));
@@ -80,7 +161,7 @@ impl BlockStore {
     }
 
     /// Retrieve all blocks from the blockstore in the form of a tuple
-    /// (`blockhash`, `block`).
+    /// (`headerhash`, `block`).
     /// Be careful as this will try to load everything in memory.
     pub fn get_all(&self) -> Result<Vec<(blake3::Hash, Block)>> {
         let mut blocks = vec![];
@@ -98,7 +179,7 @@ impl BlockStore {
 
 /// The `BlockOrderStore` is a `sled` tree storing the order of the
 /// blockchain's slots, where the key is the slot uid, and the value is
-/// the block's hash. [`BlockStore`] can be queried with this hash.
+/// the block's headers' hash. [`BlockStore`] can be queried with this hash.
 pub struct BlockOrderStore(sled::Tree);
 
 impl BlockOrderStore {
@@ -110,16 +191,15 @@ impl BlockOrderStore {
         // In case the store is empty, initialize it with the genesis block.
         if store.0.is_empty() {
             let genesis_block = Block::genesis_block(genesis_ts, genesis_data);
-            let blockhash = blake3::hash(&serialize(&genesis_block));
-            store.insert(&[genesis_block.sl], &[blockhash])?;
+            store.insert(&[0], &[genesis_block.header])?;
         }
 
         Ok(store)
     }
 
-    /// Insert a slice of slots and blockhashes into the store. With sled, the
+    /// Insert a slice of slots and headerhashes into the store. With sled, the
     /// operation is done as a batch.
-    /// The block slot is used as the key, and the blockhash is used as value.
+    /// The block slot is used as the key, and the headerhash is used as value.
     pub fn insert(&self, slots: &[u64], hashes: &[blake3::Hash]) -> Result<()> {
         assert_eq!(slots.len(), hashes.len());
         let mut batch = sled::Batch::default();
@@ -162,7 +242,7 @@ impl BlockOrderStore {
     }
 
     /// Retrieve all slots from the blockorderstore in the form of a tuple
-    /// (`slot`, `blockhash`).
+    /// (`slot`, `headerhash`).
     /// Be careful as this will try to load everything in memory.
     pub fn get_all(&self) -> Result<Vec<(u64, blake3::Hash)>> {
         let mut slots = vec![];
@@ -191,8 +271,8 @@ impl BlockOrderStore {
             if let Some(found) = self.0.get_gt(key.to_be_bytes())? {
                 let key_bytes: [u8; 8] = found.0.as_ref().try_into().unwrap();
                 key = u64::from_be_bytes(key_bytes);
-                let block_hash = deserialize(&found.1)?;
-                ret.push(block_hash);
+                let header_hash = deserialize(&found.1)?;
+                ret.push(header_hash);
                 counter += 1;
                 continue
             }
@@ -202,7 +282,7 @@ impl BlockOrderStore {
         Ok(ret)
     }
 
-    /// Fetch the last block hash in the tree, based on the `Ord`
+    /// Fetch the last block headerhash in the tree, based on the `Ord`
     /// implementation for `Vec<u8>`. This should not be able to
     /// fail because we initialize the store with the genesis block.
     pub fn get_last(&self) -> Result<(u64, blake3::Hash)> {

src/blockchain/mod.rs

@@ -13,7 +13,7 @@ use crate::{
 };
 
 pub mod blockstore;
-pub use blockstore::{BlockOrderStore, BlockStore};
+pub use blockstore::{BlockOrderStore, BlockStore, HeaderStore};
 
 pub mod metadatastore;
 pub use metadatastore::StreamletMetadataStore;
@@ -29,6 +29,8 @@ pub use txstore::TxStore;
 
 /// Structure holding all sled trees that comprise the concept of Blockchain.
 pub struct Blockchain {
+    /// Headers sled tree
+    pub headers: HeaderStore,
     /// Blocks sled tree
     pub blocks: BlockStore,
     /// Block order sled tree
@@ -46,6 +48,7 @@ pub struct Blockchain {
 impl Blockchain {
     /// Instantiate a new `Blockchain` with the given `sled` database.
     pub fn new(db: &sled::Db, genesis_ts: Timestamp, genesis_data: blake3::Hash) -> Result<Self> {
+        let headers = HeaderStore::new(db, genesis_ts, genesis_data)?;
         let blocks = BlockStore::new(db, genesis_ts, genesis_data)?;
         let order = BlockOrderStore::new(db, genesis_ts, genesis_data)?;
         let streamlet_metadata = StreamletMetadataStore::new(db, genesis_ts, genesis_data)?;
@@ -53,7 +56,15 @@ impl Blockchain {
         let nullifiers = NullifierStore::new(db)?;
         let merkle_roots = RootStore::new(db)?;
 
-        Ok(Self { blocks, order, transactions, streamlet_metadata, nullifiers, merkle_roots })
+        Ok(Self {
+            headers,
+            blocks,
+            order,
+            transactions,
+            streamlet_metadata,
+            nullifiers,
+            merkle_roots,
+        })
     }
 
     /// Insert a given slice of [`BlockInfo`] into the blockchain database.
@@ -68,16 +79,19 @@ impl Blockchain {
             // Store transactions
             let tx_hashes = self.transactions.insert(&block.txs)?;
 
+            // Store header
+            let headerhash = self.headers.insert(&[block.header.clone()])?;
+            ret.push(headerhash[0]);
+
             // Store block
-            let _block = Block::new(block.st, block.e, block.sl, tx_hashes, block.metadata.clone());
-            let blockhash = self.blocks.insert(&[_block])?;
-            ret.push(blockhash[0]);
+            let _block = Block::new(headerhash[0], tx_hashes, block.metadata.clone());
+            self.blocks.insert(&[_block])?;
 
             // Store block order
-            self.order.insert(&[block.sl], &[blockhash[0]])?;
+            self.order.insert(&[block.header.sl], &[headerhash[0]])?;
 
             // Store streamlet metadata
-            self.streamlet_metadata.insert(&[blockhash[0]], &[block.sm.clone()])?;
+            self.streamlet_metadata.insert(&[headerhash[0]], &[block.sm.clone()])?;
 
             // NOTE: The nullifiers and Merkle roots are applied in the state
             // transition apply function.
@@ -88,7 +102,7 @@ impl Blockchain {
 
     /// Check if the given [`BlockInfo`] is in the database and all trees.
     pub fn has_block(&self, block: &BlockInfo) -> Result<bool> {
-        let blockhash = match self.order.get(&[block.sl], true) {
+        let blockhash = match self.order.get(&[block.header.sl], true) {
             Ok(v) => v[0].unwrap(),
             Err(_) => return Ok(false),
         };
@@ -96,24 +110,26 @@ impl Blockchain {
         // TODO: Check if we have all transactions
 
         // Check provided info produces the same hash
-        Ok(blockhash == block.blockhash())
+        Ok(blockhash == block.header.headerhash())
     }
 
     /// Retrieve [`BlockInfo`]s by given hashes. Fails if any of them are not found.
     pub fn get_blocks_by_hash(&self, hashes: &[blake3::Hash]) -> Result<Vec<BlockInfo>> {
         let mut ret = Vec::with_capacity(hashes.len());
 
+        let headers = self.headers.get(hashes, true)?;
         let blocks = self.blocks.get(hashes, true)?;
         let metadata = self.streamlet_metadata.get(hashes, true)?;
 
-        for (i, block) in blocks.iter().enumerate() {
-            let block = block.clone().unwrap();
+        for (i, header) in headers.iter().enumerate() {
+            let header = header.clone().unwrap();
+            let block = blocks[i].clone().unwrap();
             let sm = metadata[i].clone().unwrap();
 
             let txs = self.transactions.get(&block.txs, true)?;
             let txs = txs.iter().map(|x| x.clone().unwrap()).collect();
 
-            let info = BlockInfo::new(block.st, block.e, block.sl, txs, block.metadata.clone(), sm);
+            let info = BlockInfo::new(header, txs, block.metadata.clone(), sm);
             ret.push(info);
         }
 

src/consensus/block.rs

@@ -1,10 +1,16 @@
 use std::io;
 
+use incrementalmerkletree::{bridgetree::BridgeTree, Tree};
 use log::debug;
 
-use super::{Metadata, StreamletMetadata, BLOCK_VERSION};
+use super::{
+    Metadata, StreamletMetadata, BLOCK_INFO_MAGIC_BYTES, BLOCK_MAGIC_BYTES, BLOCK_VERSION,
+};
 use crate::{
-    crypto::{address::Address, keypair::PublicKey, schnorr::Signature},
+    crypto::{
+        address::Address, constants::MERKLE_DEPTH, keypair::PublicKey, merkle_node::MerkleNode,
+        schnorr::Signature,
+    },
     impl_vec, net,
     tx::Transaction,
     util::{
@@ -14,19 +20,52 @@ use crate::{
     Result,
 };
 
-/// This struct represents a tuple of the form (`v`, `st`, `e`, `sl`, `txs`, `metadata`).
-/// The transactions here are stored as hashes, which serve as pointers to
-/// the actual transaction data in the blockchain database.
-#[derive(Debug, Clone, SerialEncodable, SerialDecodable)]
-pub struct Block {
+/// This struct represents a tuple of the form (version, state, epoch, slot, timestamp, merkle_root).
+#[derive(Debug, Clone, PartialEq, SerialEncodable, SerialDecodable)]
+pub struct Header {
     /// Block version
     pub v: u8,
     /// Previous block hash
     pub st: blake3::Hash,
     /// Epoch
     pub e: u64,
-    /// Slot uid
+    /// Slot UID
     pub sl: u64,
+    /// Block creation timestamp
+    pub timestamp: Timestamp,
+    /// Root of the transaction hashes merkle tree
+    pub root: MerkleNode,
+}
+
+impl Header {
+    pub fn new(st: blake3::Hash, e: u64, sl: u64, timestamp: Timestamp, root: MerkleNode) -> Self {
+        let v = *BLOCK_VERSION;
+        Self { v, st, e, sl, timestamp, root }
+    }
+
+    /// Generate the genesis block.
+    pub fn genesis_header(genesis_ts: Timestamp, genesis_data: blake3::Hash) -> Self {
+        let tree = BridgeTree::<MerkleNode, MERKLE_DEPTH>::new(100);
+        let root = tree.root(0).unwrap();
+
+        Self::new(genesis_data, 0, 0, genesis_ts, root)
+    }
+
+    /// Calculate the header hash
+    pub fn headerhash(&self) -> blake3::Hash {
+        blake3::hash(&serialize(self))
+    }
+}
+
+/// This struct represents a tuple of the form (`magic`, `header`, `counter`, `txs`, `metadata`).
+/// The header and transactions are stored as hashes, serving as pointers to
+/// the actual data in the sled database.
+#[derive(Debug, Clone, SerialEncodable, SerialDecodable)]
+pub struct Block {
+    /// Block magic bytes
+    pub magic: [u8; 4],
+    /// Block header hash
+    pub header: blake3::Hash,
     /// Transaction hashes
     pub txs: Vec<blake3::Hash>,
     /// Additional block information
@@ -34,28 +73,17 @@ pub struct Block {
 }
 
 impl Block {
-    pub fn new(
-        st: blake3::Hash,
-        e: u64,
-        sl: u64,
-        txs: Vec<blake3::Hash>,
-        metadata: Metadata,
-    ) -> Self {
-        let v = *BLOCK_VERSION;
-        Self { v, st, e, sl, txs, metadata }
+    pub fn new(header: blake3::Hash, txs: Vec<blake3::Hash>, metadata: Metadata) -> Self {
+        let magic = *BLOCK_MAGIC_BYTES;
+        Self { magic, header, txs, metadata }
     }
 
     /// Generate the genesis block.
     pub fn genesis_block(genesis_ts: Timestamp, genesis_data: blake3::Hash) -> Self {
-        let metadata =
-            Metadata::new(genesis_ts, String::from("proof"), String::from("r"), String::from("s"));
-
-        Self::new(genesis_data, 0, 0, vec![], metadata)
-    }
+        let header = Header::genesis_header(genesis_ts, genesis_data);
+        let metadata = Metadata::new(String::from("proof"), String::from("r"), String::from("s"));
 
-    /// Calculate the block hash
-    pub fn blockhash(&self) -> blake3::Hash {
-        blake3::hash(&serialize(self))
+        Self::new(header.headerhash(), vec![], metadata)
     }
 }
 
@@ -64,7 +92,7 @@ impl Block {
 pub struct BlockOrder {
     /// Slot UID
     pub sl: u64,
-    /// Blockhash of that slot
+    /// Block headerhash of that slot
     pub block: blake3::Hash,
 }
 
@@ -77,14 +105,10 @@ impl net::Message for BlockOrder {
 /// Structure representing full block data.
 #[derive(Debug, Clone, SerialEncodable, SerialDecodable)]
 pub struct BlockInfo {
-    /// Block version
-    pub v: u8,
-    /// Previous block hash
-    pub st: blake3::Hash,
-    /// Epoch
-    pub e: u64,
-    /// Slot uid
-    pub sl: u64,
+    /// BlockInfo magic bytes
+    pub magic: [u8; 4],
+    /// Block header data
+    pub header: Header,
     /// Transactions payload
     pub txs: Vec<Transaction>,
     /// Additional proposal information
@@ -95,28 +119,13 @@ pub struct BlockInfo {
 
 impl BlockInfo {
     pub fn new(
-        st: blake3::Hash,
-        e: u64,
-        sl: u64,
+        header: Header,
         txs: Vec<Transaction>,
         metadata: Metadata,
         sm: StreamletMetadata,
     ) -> Self {
-        let v = *BLOCK_VERSION;
-        Self { v, st, e, sl, txs, metadata, sm }
-    }
-
-    /// Calculate the block hash
-    pub fn blockhash(&self) -> blake3::Hash {
-        let block: Block = self.clone().into();
-        block.blockhash()
-    }
-}
-
-impl From<BlockInfo> for Block {
-    fn from(b: BlockInfo) -> Self {
-        let txids = b.txs.iter().map(|x| blake3::hash(&serialize(x))).collect();
-        Self { v: b.v, st: b.st, e: b.e, sl: b.sl, txs: txids, metadata: b.metadata }
+        let magic = *BLOCK_INFO_MAGIC_BYTES;
+        Self { magic, header, txs, metadata, sm }
     }
 }
 
@@ -160,43 +169,14 @@ impl BlockProposal {
         public_key: PublicKey,
         signature: Signature,
         address: Address,
-        st: blake3::Hash,
-        e: u64,
-        sl: u64,
+        header: Header,
         txs: Vec<Transaction>,
         metadata: Metadata,
         sm: StreamletMetadata,
     ) -> Self {
-        let block = BlockInfo::new(st, e, sl, txs, metadata, sm);
+        let block = BlockInfo::new(header, txs, metadata, sm);
         Self { public_key, signature, address, block }
     }
-
-    /// Produce proposal hash using `st`, `e`, `sl`, `txs`, and `metadata`.
-    pub fn hash(&self) -> blake3::Hash {
-        Self::to_proposal_hash(
-            self.block.st,
-            self.block.e,
-            self.block.sl,
-            &self.block.txs,
-            &self.block.metadata,
-        )
-    }
-
-    /// Generate a proposal hash using provided `st`, `e`, `sl`, `txs`, and `metadata`.
-    pub fn to_proposal_hash(
-        st: blake3::Hash,
-        e: u64,
-        sl: u64,
-        transactions: &[Transaction],
-        metadata: &Metadata,
-    ) -> blake3::Hash {
-        let mut txs = Vec::with_capacity(transactions.len());
-        for tx in transactions {
-            txs.push(blake3::hash(&serialize(tx)));
-        }
-
-        blake3::hash(&serialize(&Block::new(st, e, sl, txs, metadata.clone())))
-    }
 }
 
 impl PartialEq for BlockProposal {
@@ -204,9 +184,7 @@ impl PartialEq for BlockProposal {
         self.public_key == other.public_key &&
             self.signature == other.signature &&
             self.address == other.address &&
-            self.block.st == other.block.st &&
-            self.block.e == other.block.e &&
-            self.block.sl == other.block.sl &&
+            self.block.header == other.block.header &&
             self.block.txs == other.block.txs &&
             self.block.metadata == other.block.metadata
     }
@@ -243,13 +221,15 @@ impl ProposalChain {
     /// excluding the genesis block proposal.
     /// Additional validity rules can be applied.
     pub fn check_proposal(&self, proposal: &BlockProposal, previous: &BlockProposal) -> bool {
-        if proposal.block.st == self.genesis_block {
+        if proposal.block.header.st == self.genesis_block {
             debug!("check_proposal(): Genesis block proposal provided.");
             return false
         }
 
-        let prev_hash = previous.hash();
-        if proposal.block.st != prev_hash || proposal.block.sl <= previous.block.sl {
+        let prev_hash = previous.block.header.headerhash();
+        if proposal.block.header.st != prev_hash ||
+            proposal.block.header.sl <= previous.block.header.sl
+        {
             debug!("check_proposal(): Provided proposal is invalid.");
             return false
         }

src/consensus/metadata.rs

@@ -1,29 +1,10 @@
 use super::{Participant, Vote};
-use crate::util::{
-    serial::{SerialDecodable, SerialEncodable},
-    time::Timestamp,
-};
-
-/// This struct represents additional [`Block`](super::Block) information used by
-/// the consensus protocol
-#[derive(Debug, Clone, PartialEq, SerialEncodable, SerialDecodable)]
-pub struct Metadata {
-    /// Block creation timestamp
-    pub timestamp: Timestamp,
-    /// Block information used by the Ouroboros Praos consensus
-    pub om: OuroborosMetadata,
-}
-
-impl Metadata {
-    pub fn new(timestamp: Timestamp, proof: String, r: String, s: String) -> Self {
-        Self { timestamp, om: OuroborosMetadata::new(proof, r, s) }
-    }
-}
+use crate::util::serial::{SerialDecodable, SerialEncodable};
 
 /// This struct represents [`Block`](super::Block) information used by the Ouroboros
 /// Praos consensus protocol.
 #[derive(Debug, Clone, PartialEq, SerialEncodable, SerialDecodable)]
-pub struct OuroborosMetadata {
+pub struct Metadata {
     /// Proof that the stakeholder is the block owner
     pub proof: String,
     /// Random seed for VRF
@@ -32,7 +13,7 @@ pub struct OuroborosMetadata {
     pub s: String,
 }
 
-impl OuroborosMetadata {
+impl Metadata {
     pub fn new(proof: String, r: String, s: String) -> Self {
         Self { proof, r, s }
     }

src/consensus/mod.rs

@@ -1,6 +1,6 @@
 /// Block definition
 pub mod block;
-pub use block::{Block, BlockInfo, BlockProposal, ProposalChain};
+pub use block::{Block, BlockInfo, BlockProposal, Header, ProposalChain};
 
 /// Consensus metadata
 pub mod metadata;
@@ -43,4 +43,10 @@ lazy_static! {
 
     /// Block version number
     pub static ref BLOCK_VERSION: u8 = 1;
+
+    /// Block magic bytes
+    pub static ref BLOCK_MAGIC_BYTES: [u8; 4] = [0x11, 0x6d, 0x75, 0x1f];
+
+    /// Block info magic bytes
+    pub static ref BLOCK_INFO_MAGIC_BYTES: [u8; 4] = [0x90, 0x44, 0xf1, 0xf6];
 }

src/consensus/proto/protocol_sync.rs

@@ -108,7 +108,10 @@ impl ProtocolSync {
                 }
             };
 
-            info!("ProtocolSync::handle_receive_block() Received block {}", info.blockhash());
+            info!(
+                "ProtocolSync::handle_receive_block() Received block {}",
+                info.header.headerhash()
+            );
 
             // We block here if there's a pending validation, otherwise we might
             // apply the same block twice.

src/consensus/state.rs

@@ -7,18 +7,22 @@ use std::{
 
 use async_std::sync::{Arc, Mutex, RwLock};
 use chrono::{NaiveDateTime, Utc};
+use incrementalmerkletree::{bridgetree::BridgeTree, Tree};
 use lazy_init::Lazy;
 use log::{debug, error, info, warn};
 use rand::rngs::OsRng;
 
 use super::{
-    Block, BlockInfo, BlockProposal, Metadata, Participant, ProposalChain, StreamletMetadata, Vote,
+    Block, BlockInfo, BlockProposal, Header, Metadata, Participant, ProposalChain,
+    StreamletMetadata, Vote,
 };
 use crate::{
     blockchain::Blockchain,
     crypto::{
         address::Address,
+        constants::MERKLE_DEPTH,
         keypair::{PublicKey, SecretKey},
+        merkle_node::MerkleNode,
         schnorr::{SchnorrPublic, SchnorrSecret},
     },
     net,
@@ -206,8 +210,8 @@ impl ValidatorState {
         let mut slot = 0;
         for chain in &self.consensus.proposals {
             for proposal in &chain.proposals {
-                if proposal.block.sl > slot {
-                    slot = proposal.block.sl;
+                if proposal.block.header.sl > slot {
+                    slot = proposal.block.header.sl;
                 }
             }
         }
@@ -271,30 +275,29 @@ impl ValidatorState {
         let (prev_hash, index) = self.longest_notarized_chain_last_hash().unwrap();
         let unproposed_txs = self.unproposed_txs(index);
 
-        let metadata = Metadata::new(
-            Timestamp::current_time(),
-            String::from("proof"),
-            String::from("r"),
-            String::from("s"),
-        );
+        let mut tree = BridgeTree::<MerkleNode, MERKLE_DEPTH>::new(100);
+        for tx in &unproposed_txs {
+            for output in &tx.outputs {
+                tree.append(&MerkleNode::from_coin(&output.revealed.coin));
+                tree.witness();
+            }
+        }
+        let root = tree.root(0).unwrap();
+
+        let header =
+            Header::new(prev_hash, self.slot_epoch(slot), slot, Timestamp::current_time(), root);
+
+        let metadata = Metadata::new(String::from("proof"), String::from("r"), String::from("s"));
 
         let sm = StreamletMetadata::new(self.consensus.participants.values().cloned().collect());
-        let prop = BlockProposal::to_proposal_hash(
-            prev_hash,
-            self.slot_epoch(slot),
-            slot,
-            &unproposed_txs,
-            &metadata,
-        );
-        let signed_proposal = self.secret.sign(&prop.as_bytes()[..]);
+
+        let signed_proposal = self.secret.sign(&header.headerhash().as_bytes()[..]);
 
         Ok(Some(BlockProposal::new(
             self.public,
             signed_proposal,
             self.address,
-            prev_hash,
-            self.slot_epoch(slot),
-            slot,
+            header,
             unproposed_txs,
             metadata,
             sm,
@@ -344,7 +347,7 @@ impl ValidatorState {
         }
 
         let hash = match longest_notarized_chain {
-            Some(chain) => chain.proposals.last().unwrap().hash(),
+            Some(chain) => chain.proposals.last().unwrap().block.header.headerhash(),
             None => self.blockchain.last()?.1,
         };
 
@@ -377,17 +380,10 @@ impl ValidatorState {
             return Ok(None)
         }
 
-        if !proposal.public_key.verify(
-            BlockProposal::to_proposal_hash(
-                proposal.block.st,
-                proposal.block.e,
-                proposal.block.sl,
-                &proposal.block.txs,
-                &proposal.block.metadata,
-            )
-            .as_bytes(),
-            &proposal.signature,
-        ) {
+        if !proposal
+            .public_key
+            .verify(proposal.block.header.headerhash().as_bytes(), &proposal.signature)
+        {
             warn!("Proposer ({}) signature could not be verified", proposal.address.to_string());
             return Ok(None)
         }
@@ -403,7 +399,7 @@ impl ValidatorState {
         let mut proposal = proposal.clone();
 
         // Generate proposal hash
-        let proposal_hash = proposal.hash();
+        let proposal_hash = proposal.block.header.headerhash();
 
         // Add orphan votes
         let mut orphans = Vec::new();
@@ -446,7 +442,7 @@ impl ValidatorState {
             self.public,
             signed_hash,
             proposal_hash,
-            proposal.block.sl,
+            proposal.block.header.sl,
             self.address,
         )))
     }
@@ -467,17 +463,21 @@ impl ValidatorState {
         let mut fork = None;
         for (index, chain) in self.consensus.proposals.iter().enumerate() {
             let last = chain.proposals.last().unwrap();
-            let hash = last.hash();
-            if proposal.block.st == hash && proposal.block.sl > last.block.sl {
+            let hash = last.block.header.headerhash();
+            if proposal.block.header.st == hash && proposal.block.header.sl > last.block.header.sl {
                 return Ok(index as i64)
             }
 
-            if proposal.block.st == last.block.st && proposal.block.sl == last.block.sl {
+            if proposal.block.header.st == last.block.header.st &&
+                proposal.block.header.sl == last.block.header.sl
+            {
                 debug!("find_extended_chain_index(): Proposal already received");
                 return Ok(-2)
             }
 
-            if proposal.block.st == last.block.st && proposal.block.sl > last.block.sl {
+            if proposal.block.header.st == last.block.header.st &&
+                proposal.block.header.sl > last.block.header.sl
+            {
                 fork = Some(chain.clone());
             }
         }
@@ -496,7 +496,7 @@ impl ValidatorState {
         }
 
         let (last_sl, last_block) = self.blockchain.last()?;
-        if proposal.block.st != last_block || proposal.block.sl <= last_sl {
+        if proposal.block.header.st != last_block || proposal.block.header.sl <= last_sl {
             debug!("find_extended_chain_index(): Proposal doesn't extend any known chain");
             return Ok(-2)
         }
@@ -623,7 +623,7 @@ impl ValidatorState {
     ) -> Result<Option<(&mut BlockProposal, i64)>> {
         for (index, chain) in &mut self.consensus.proposals.iter_mut().enumerate() {
             for proposal in chain.proposals.iter_mut().rev() {
-                let proposal_hash = proposal.hash();
+                let proposal_hash = proposal.block.header.headerhash();
                 if vote_proposal == &proposal_hash {
                     return Ok(Some((proposal, index as i64)))
                 }
@@ -709,12 +709,12 @@ impl ValidatorState {
         }
 
         let last_block = *blockhashes.last().unwrap();
-        let last_sl = finalized.last().unwrap().sl;
+        let last_sl = finalized.last().unwrap().header.sl;
 
         let mut dropped = vec![];
         for chain in self.consensus.proposals.iter() {
             let first = chain.proposals.first().unwrap();
-            if first.block.st != last_block || first.block.sl <= last_sl {
+            if first.block.header.st != last_block || first.block.header.sl <= last_sl {
                 dropped.push(chain.clone());
             }
         }

src/error.rs

@@ -216,6 +216,9 @@ pub enum Error {
     #[error("Transaction {0} not found in database")]
     TransactionNotFound(String),
 
+    #[error("Header {0} not found in database")]
+    HeaderNotFound(String),
+
     #[error("Block {0} not found in database")]
     BlockNotFound(String),