/* This file is part of DarkFi (https://dark.fi) * * Copyright (C) 2020-2026 Dyne.org foundation * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ use std::{ collections::{HashMap, HashSet}, sync::Arc, }; use num_bigint::BigUint; use smol::{channel::Receiver, lock::RwLock}; use tinyjson::JsonValue; use tracing::{debug, error, info}; use darkfi::{ blockchain::{BlockDifficulty, BlockchainOverlay, HeaderHash}, net::ChannelPtr, util::{encoding::base64, time::Timestamp}, validator::{ consensus::{Fork, Proposal}, pow::PoWModule, utils::{best_fork_index, header_rank}, verification::verify_fork_proposal, Validator, ValidatorPtr, }, Error::{Custom, DatabaseError, PoWInvalidOutHash, ProposalAlreadyExists}, Result, }; use darkfi_serial::serialize_async; use crate::{ proto::{ ForkHeaderHashRequest, ForkHeaderHashResponse, ForkHeadersRequest, ForkHeadersResponse, ForkProposalsRequest, ForkProposalsResponse, ForkSyncRequest, ForkSyncResponse, ProposalMessage, BATCH, }, DarkfiNodePtr, }; /// Background task to handle unknown proposals. pub async fn handle_unknown_proposals( receiver: Receiver<(Proposal, u32)>, unknown_proposals: Arc>>, unknown_proposals_channels: Arc>>, node: DarkfiNodePtr, ) -> Result<()> { debug!(target: "darkfid::task::handle_unknown_proposal", "START"); loop { // Wait for a new unknown proposal trigger let (proposal, channel) = match receiver.recv().await { Ok(m) => m, Err(e) => { debug!( target: "darkfid::task::handle_unknown_proposal", "recv fail: {e}" ); continue } }; // Check if proposal exists in our queue let lock = unknown_proposals.read().await; let contains_proposal = lock.contains(proposal.hash.inner()); drop(lock); if !contains_proposal { debug!( target: "darkfid::task::handle_unknown_proposal", "Proposal {} is not in our unknown proposals queue.", proposal.hash, ); continue }; // Increase channel counter let mut lock = unknown_proposals_channels.write().await; let channel_counter = if let Some((counter, timestamp)) = lock.get_mut(&channel) { *counter += 1; *timestamp = Timestamp::current_time().inner(); *counter } else { lock.insert(channel, (1, Timestamp::current_time().inner())); 1 }; drop(lock); // Handle the unknown proposal if handle_unknown_proposal(&node, channel, &proposal).await { // Ban channel if it exceeds 5 consecutive unknown proposals if channel_counter > 5 { if let Some(channel) = node.p2p_handler.p2p.get_channel(channel) { channel.ban().await; } unknown_proposals_channels.write().await.remove(&channel); } }; // Remove proposal from the queue let mut lock = unknown_proposals.write().await; lock.remove(proposal.hash.inner()); drop(lock); } } /// Background task to handle an unknown proposal. /// Returns a boolean flag indicate if we should ban the channel. async fn handle_unknown_proposal(node: &DarkfiNodePtr, channel: u32, proposal: &Proposal) -> bool { // If proposal fork chain was not found, we ask our peer for its sequence debug!(target: "darkfid::task::handle_unknown_proposal", "Asking peer for fork sequence"); let Some(channel) = node.p2p_handler.p2p.get_channel(channel) else { debug!(target: "darkfid::task::handle_unknown_proposal", "Channel {channel} wasn't found."); return false }; // Communication setup let Ok(response_sub) = channel.subscribe_msg::().await else { debug!(target: "darkfid::task::handle_unknown_proposal", "Failure during `ForkSyncResponse` communication setup with peer: {channel:?}"); return true }; // Grab last known block to create the request and execute it let last = match node.validator.read().await.blockchain.last() { Ok(l) => l, Err(e) => { error!(target: "darkfid::task::handle_unknown_proposal", "Blockchain last retriaval failed: {e}"); return false } }; let request = ForkSyncRequest { tip: last.1, fork_tip: Some(proposal.hash) }; if let Err(e) = channel.send(&request).await { debug!(target: "darkfid::task::handle_unknown_proposal", "Channel send failed: {e}"); return true }; let comms_timeout = node .p2p_handler .p2p .settings() .read_arc() .await .outbound_connect_timeout(channel.address().scheme()); // Node waits for response let response = match response_sub.receive_with_timeout(comms_timeout).await { Ok(r) => r, Err(e) => { debug!(target: "darkfid::task::handle_unknown_proposal", "Asking peer for fork sequence failed: {e}"); return true } }; debug!(target: "darkfid::task::handle_unknown_proposal", "Peer response: {response:?}"); // Verify and store retrieved proposals debug!(target: "darkfid::task::handle_unknown_proposal", "Processing received proposals"); // Response should not be empty if response.proposals.is_empty() { debug!(target: "darkfid::task::handle_unknown_proposal", "Peer responded with empty sequence, node might be out of sync!"); return handle_reorg(node, &(&channel, &comms_timeout), proposal).await } // Sequence length must correspond to requested height if response.proposals.len() as u32 != proposal.block.header.height - last.0 { debug!(target: "darkfid::task::handle_unknown_proposal", "Response sequence length is erroneous"); return handle_reorg(node, &(&channel, &comms_timeout), proposal).await } // First proposal must extend canonical if response.proposals[0].block.header.previous != last.1 { debug!(target: "darkfid::task::handle_unknown_proposal", "Response sequence doesn't extend canonical"); return handle_reorg(node, &(&channel, &comms_timeout), proposal).await } // Last proposal must be the same as the one requested if response.proposals.last().unwrap().hash != proposal.hash { debug!(target: "darkfid::task::handle_unknown_proposal", "Response sequence doesn't correspond to requested tip"); return handle_reorg(node, &(&channel, &comms_timeout), proposal).await } // Process response proposals for proposal in &response.proposals { // Append proposal match node.validator.write().await.append_proposal(proposal).await { Ok(()) => { /* Do nothing */ } // Skip already existing proposals Err(ProposalAlreadyExists) => continue, Err(e) => { debug!( target: "darkfid::task::handle_unknown_proposal", "Error while appending response proposal: {e}" ); break; } }; // Broadcast proposal to rest nodes let message = ProposalMessage(proposal.clone()); node.p2p_handler.p2p.broadcast_with_exclude(&message, &[channel.address().clone()]).await; // Notify proposals subscriber let enc_prop = JsonValue::String(base64::encode(&serialize_async(proposal).await)); node.subscribers.get("proposals").unwrap().notify(vec![enc_prop].into()).await; } false } /// Auxiliary function to handle a potential reorg. We first find our /// last common block with the peer, then grab the header sequence from /// that block until the proposal and check if it ranks higher than our /// current best ranking fork, to perform a reorg. /// /// Returns a boolean flag indicate if we should ban the channel. /// /// Note: Always remember to purge new trees from the database if not /// needed. // TODO: We keep everything in memory which can result in OOM for a // valid long fork. We could use some disk space to store stuff. async fn handle_reorg( // Node pointer node: &DarkfiNodePtr, // Peer channel and its communications timeout channel: &(&ChannelPtr, &u64), // Peer fork proposal proposal: &Proposal, ) -> bool { info!(target: "darkfid::task::handle_reorg", "Checking for potential reorg from proposal {} - {} by peer: {:?}", proposal.hash, proposal.block.header.height, channel.0); // Check if genesis proposal was provided if proposal.block.header.height == 0 { debug!(target: "darkfid::task::handle_reorg", "Peer send a genesis proposal, skipping..."); return true } // Find last common header and its sequence, going backwards from // the proposal. let (last_common_height, last_common_hash, peer_header_hashes) = match retrieve_peer_header_hashes(&node.validator, channel, proposal).await { Ok(t) => t, Err(DatabaseError(e)) => { error!(target: "darkfid::task::handle_reorg", "Internal error while retrieving peer headers hashes: {e}"); return false } Err(e) => { error!(target: "darkfid::task::handle_reorg", "Retrieving peer headers hashes failed: {e}"); return true } }; // Create a new PoW module from last common height let validator = node.validator.read().await; let module = match PoWModule::new( validator.consensus.blockchain.clone(), validator.consensus.module.target, validator.consensus.module.fixed_difficulty.clone(), Some(last_common_height + 1), ) { Ok(m) => m, Err(e) => { error!(target: "darkfid::task::handle_reorg", "PoWModule generation failed: {e}"); return false } }; // Grab last common height ranks let last_difficulty = match last_common_height { 0 => { let genesis_timestamp = match validator.blockchain.genesis_block() { Ok(b) => b.header.timestamp, Err(e) => { error!(target: "darkfid::task::handle_reorg", "Retrieving genesis block failed: {e}"); return false } }; BlockDifficulty::genesis(genesis_timestamp) } _ => match validator.blockchain.blocks.get_difficulty(&[last_common_height], true) { Ok(d) => d[0].clone().unwrap(), Err(e) => { error!(target: "darkfid::task::handle_reorg", "Retrieving block difficulty failed: {e}"); return false } }, }; drop(validator); // Retrieve the headers of the hashes sequence and its ranking let (targets_rank, hashes_rank) = match retrieve_peer_headers_sequence_ranking( (&last_common_height, &last_common_hash, &module, &last_difficulty), channel, proposal, &peer_header_hashes, ) .await { Ok(p) => p, Err(DatabaseError(e)) => { error!(target: "darkfid::task::handle_reorg", "Internal error while retrieving peer headers: {e}"); return false } Err(e) => { error!(target: "darkfid::task::handle_reorg", "Retrieving peer headers failed: {e}"); return true } }; // Grab the validator lock so no other proposal gets processed // while we are verifying the sequence. let mut validator = node.validator.write().await; // Check if the sequence ranks higher than our current best fork let index = match best_fork_index(&validator.consensus.forks) { Ok(i) => i, Err(e) => { debug!(target: "darkfid::task::handle_reorg", "Retrieving best fork index failed: {e}"); return false } }; let best_fork = &validator.consensus.forks[index]; if targets_rank < best_fork.targets_rank || (targets_rank == best_fork.targets_rank && hashes_rank <= best_fork.hashes_rank) { info!(target: "darkfid::task::handle_reorg", "Peer sequence ranks lower than our current best fork, skipping..."); return true } // Generate the peer fork and retrieve its ranking let mut peer_fork = match retrieve_peer_fork( &validator, (&last_common_height, &module, &last_difficulty), channel, proposal, &peer_header_hashes, ) .await { Ok(p) => p, Err(DatabaseError(e)) => { error!(target: "darkfid::task::handle_reorg", "Internal error while retrieving peer fork: {e}"); return false } Err(e) => { error!(target: "darkfid::task::handle_reorg", "Retrieving peer fork failed: {e}"); return true } }; // Check if the peer fork ranks higher than our current best fork if peer_fork.targets_rank < best_fork.targets_rank || (peer_fork.targets_rank == best_fork.targets_rank && peer_fork.hashes_rank <= best_fork.hashes_rank) { info!(target: "darkfid::task::handle_reorg", "Peer fork ranks lower than our current best fork, skipping..."); return true } // Execute the reorg info!(target: "darkfid::task::handle_reorg", "Peer fork ranks higher than our current best fork, executing reorg..."); if let Err(e) = validator.blockchain.reset_to_height(last_common_height) { error!(target: "darkfid::task::handle_reorg", "Applying full inverse diff failed: {e}"); return false }; // Update fork diffs to forward-only ones let overlay = match BlockchainOverlay::new(&validator.blockchain) { Ok(o) => o, Err(e) => { error!(target: "darkfid::task::handle_reorg", "Generating a new blockchain overlay failed: {e}"); return false } }; let mut diffs = Vec::with_capacity(peer_fork.diffs.len()); for diff in peer_fork.diffs { let overlay = overlay.lock().unwrap(); let mut overlay = overlay.overlay.lock().unwrap(); if let Err(e) = overlay.add_diff(&diff) { error!(target: "darkfid::task::handle_reorg", "Applying peer fork diff failed: {e}"); return false } match overlay.diff(&diffs) { Ok(diff) => diffs.push(diff), Err(e) => { error!(target: "darkfid::task::handle_reorg", "Generate clean state inverse diff failed: {e}"); return false } } } peer_fork.overlay = overlay; peer_fork.diffs = diffs; // Update validator consensus state validator.consensus.module = module; validator.consensus.forks = vec![peer_fork]; // Check if we can confirm anything and broadcast them let confirmed = match validator.confirmation().await { Ok(f) => f, Err(e) => { error!(target: "darkfid::task::handle_reorg", "Confirmation failed: {e}"); return false } }; // Refresh mining registry if let Err(e) = node.registry.state.write().await.refresh(&validator).await { error!(target: "darkfid::task::handle_reorg", "Failed refreshing mining block templates: {e}") } if !confirmed.is_empty() { let mut notif_blocks = Vec::with_capacity(confirmed.len()); for block in confirmed { notif_blocks.push(JsonValue::String(base64::encode(&serialize_async(&block).await))); } node.subscribers.get("blocks").unwrap().notify(JsonValue::Array(notif_blocks)).await; } // Broadcast proposal to the network let message = ProposalMessage(proposal.clone()); node.p2p_handler.p2p.broadcast(&message).await; // Notify proposals subscriber let enc_prop = JsonValue::String(base64::encode(&serialize_async(proposal).await)); node.subscribers.get("proposals").unwrap().notify(vec![enc_prop].into()).await; false } /// Auxiliary function to retrieve the last common header and height, /// along with the headers sequence up to provided peer proposal. async fn retrieve_peer_header_hashes( // Validator pointer validator: &ValidatorPtr, // Peer channel and its communications timeout channel: &(&ChannelPtr, &u64), // Peer fork proposal proposal: &Proposal, ) -> Result<(u32, HeaderHash, Vec)> { // Communication setup let response_sub = channel.0.subscribe_msg::().await?; // Keep track of received header hashes sequence let mut peer_header_hashes = vec![]; // Find last common header, going backwards from the proposal let mut previous_height = proposal.block.header.height; let mut previous_hash = proposal.hash; for height in (0..proposal.block.header.height).rev() { // Request peer header hash for this height let request = ForkHeaderHashRequest { height, fork_header: proposal.hash }; channel.0.send(&request).await?; // Node waits for response let response = response_sub.receive_with_timeout(*channel.1).await?; debug!(target: "darkfid::task::handle_reorg", "Peer response: {response:?}"); // Check if peer returned a header let Some(peer_header) = response.fork_header else { return Err(Custom(String::from("Peer responded with an empty header"))) }; // Check if we know this header let headers = match validator.read().await.blockchain.blocks.get_order(&[height], false) { Ok(h) => h, Err(e) => return Err(DatabaseError(format!("Retrieving headers failed: {e}"))), }; match headers[0] { Some(known_header) => { if known_header == peer_header { previous_height = height; previous_hash = known_header; break } // Since we retrieve in right -> left order we push them in reverse order peer_header_hashes.insert(0, peer_header); } None => peer_header_hashes.insert(0, peer_header), } } Ok((previous_height, previous_hash, peer_header_hashes)) } /// Auxiliary function to retrieve provided peer headers hashes /// sequence and its ranking, based on provided last common /// information. async fn retrieve_peer_headers_sequence_ranking( // Last common header, PoW module and difficulty last_common_info: (&u32, &HeaderHash, &PoWModule, &BlockDifficulty), // Peer channel and its communications timeout channel: &(&ChannelPtr, &u64), // Peer fork trigger proposal proposal: &Proposal, // Peer header hashes sequence header_hashes: &[HeaderHash], ) -> Result<(BigUint, BigUint)> { // Communication setup let response_sub = channel.0.subscribe_msg::().await?; // Retrieve the headers of the hashes sequence, in batches, keeping track of the sequence ranking info!(target: "darkfid::task::handle_reorg", "Retrieving {} headers from peer...", header_hashes.len()); let mut previous_height = *last_common_info.0; let mut previous_hash = *last_common_info.1; let mut module = last_common_info.2.clone(); let mut targets_rank = last_common_info.3.ranks.targets_rank.clone(); let mut hashes_rank = last_common_info.3.ranks.hashes_rank.clone(); let mut batch = Vec::with_capacity(BATCH); let mut total_processed = 0; for (index, hash) in header_hashes.iter().enumerate() { // Add hash in batch sequence batch.push(*hash); // Check if batch is full so we can send it if batch.len() < BATCH && index != header_hashes.len() - 1 { continue } // Request peer headers let request = ForkHeadersRequest { headers: batch.clone(), fork_header: proposal.hash }; channel.0.send(&request).await?; // Node waits for response let response = response_sub.receive_with_timeout(*channel.1).await?; debug!(target: "darkfid::task::handle_reorg", "Peer response: {response:?}"); // Response sequence must be the same length as the one requested if response.headers.len() != batch.len() { return Err(Custom(String::from( "Peer responded with a different headers sequence length", ))) } // Process retrieved headers for (peer_header_index, peer_header) in response.headers.iter().enumerate() { let peer_header_hash = peer_header.hash(); debug!(target: "darkfid::task::handle_reorg", "Processing header: {peer_header_hash} - {}", peer_header.height); // Validate its the header we requested if peer_header_hash != batch[peer_header_index] { return Err(Custom(format!( "Peer responded with a differend header: {} - {peer_header_hash}", batch[peer_header_index] ))) } // Validate sequence is correct if peer_header.previous != previous_hash || peer_header.height != previous_height + 1 { return Err(Custom(String::from("Invalid header sequence detected"))) } // Verify header hash and calculate its rank let (next_difficulty, target_distance_sq, hash_distance_sq) = match header_rank(&mut module, peer_header) { Ok(tuple) => tuple, Err(PoWInvalidOutHash) => return Err(PoWInvalidOutHash), Err(e) => { return Err(DatabaseError(format!("Computing header rank failed: {e}"))) } }; // Update sequence ranking targets_rank += target_distance_sq.clone(); hashes_rank += hash_distance_sq.clone(); // Update PoW headers module module.append(peer_header, &next_difficulty)?; // Set previous header previous_height = peer_header.height; previous_hash = peer_header_hash; } total_processed += response.headers.len(); info!(target: "darkfid::task::handle_reorg", "Headers received and verified: {total_processed}/{}", header_hashes.len()); // Reset batch batch = Vec::with_capacity(BATCH); } // Validate trigger proposal header sequence is correct if proposal.block.header.previous != previous_hash || proposal.block.header.height != previous_height + 1 { return Err(Custom(String::from("Invalid header sequence detected"))) } // Verify trigger proposal header hash and calculate its rank let (_, target_distance_sq, hash_distance_sq) = match header_rank(&mut module, &proposal.block.header) { Ok(tuple) => tuple, Err(PoWInvalidOutHash) => return Err(PoWInvalidOutHash), Err(e) => return Err(DatabaseError(format!("Computing header rank failed: {e}"))), }; // Update sequence ranking targets_rank += target_distance_sq.clone(); hashes_rank += hash_distance_sq.clone(); Ok((targets_rank, hashes_rank)) } /// Auxiliary function to generate provided peer headers hashes fork /// and its ranking, based on provided last common information. async fn retrieve_peer_fork( // Validator pointer validator: &Validator, // Last common header height, PoW module and difficulty last_common_info: (&u32, &PoWModule, &BlockDifficulty), // Peer channel and its communications timeout channel: &(&ChannelPtr, &u64), // Peer fork trigger proposal proposal: &Proposal, // Peer header hashes sequence header_hashes: &[HeaderHash], ) -> Result { // Communication setup let response_sub = channel.0.subscribe_msg::().await?; // Create a fork from last common height let mut peer_fork = match Fork::new(validator.consensus.blockchain.clone(), last_common_info.1.clone()).await { Ok(f) => f, Err(e) => return Err(DatabaseError(format!("Generating peer fork failed: {e}"))), }; peer_fork.targets_rank = last_common_info.2.ranks.targets_rank.clone(); peer_fork.hashes_rank = last_common_info.2.ranks.hashes_rank.clone(); // Grab all state inverse diffs after last common height, and add them to the fork let inverse_diffs = match validator.blockchain.blocks.get_state_inverse_diffs_after(*last_common_info.0) { Ok(i) => i, Err(e) => { return Err(DatabaseError(format!("Retrieving state inverse diffs failed: {e}"))) } }; for inverse_diff in inverse_diffs.iter().rev() { let result = peer_fork.overlay.lock().unwrap().overlay.lock().unwrap().add_diff(inverse_diff); if let Err(e) = result { return Err(DatabaseError(format!("Applying state inverse diff failed: {e}"))) } } // Grab current overlay diff and use it as the first diff of the // peer fork, so all consecutive diffs represent just the proposal // changes. let diff = peer_fork.overlay.lock().unwrap().overlay.lock().unwrap().diff(&[]); let diff = match diff { Ok(d) => d, Err(e) => { return Err(DatabaseError(format!("Generate full state inverse diff failed: {e}"))) } }; peer_fork.diffs = vec![diff]; // Retrieve the proposals of the hashes sequence, in batches info!(target: "darkfid::task::handle_reorg", "Peer sequence ranks higher than our current best fork, retrieving {} proposals from peer...", header_hashes.len()); let mut batch = Vec::with_capacity(BATCH); let mut total_processed = 0; for (index, hash) in header_hashes.iter().enumerate() { // Add hash in batch sequence batch.push(*hash); // Check if batch is full so we can send it if batch.len() < BATCH && index != header_hashes.len() - 1 { continue } // Request peer proposals let request = ForkProposalsRequest { headers: batch.clone(), fork_header: proposal.hash }; channel.0.send(&request).await?; // Node waits for response let response = response_sub.receive_with_timeout(*channel.1).await?; debug!(target: "darkfid::task::handle_reorg", "Peer response: {response:?}"); // Response sequence must be the same length as the one requested if response.proposals.len() != batch.len() { return Err(Custom(String::from( "Peer responded with a different proposals sequence length", ))) } // Process retrieved proposal for (peer_proposal_index, peer_proposal) in response.proposals.iter().enumerate() { info!(target: "darkfid::task::handle_reorg", "Processing proposal: {} - {}", peer_proposal.hash, peer_proposal.block.header.height); // Validate its the proposal we requested if peer_proposal.hash != batch[peer_proposal_index] { return Err(Custom(format!( "Peer responded with a differend proposal: {} - {}", batch[peer_proposal_index], peer_proposal.hash ))) } // Verify proposal verify_fork_proposal(&mut peer_fork, peer_proposal, validator.verify_fees).await?; // Append proposal peer_fork.append_proposal(peer_proposal).await?; } total_processed += response.proposals.len(); info!(target: "darkfid::task::handle_reorg", "Proposals received and verified: {total_processed}/{}", header_hashes.len()); // Reset batch batch = Vec::with_capacity(BATCH); } // Verify trigger proposal verify_fork_proposal(&mut peer_fork, proposal, validator.verify_fees).await?; // Append trigger proposal peer_fork.append_proposal(proposal).await?; // Remove the reorg diff from the fork peer_fork.diffs.remove(0); Ok(peer_fork) }