/* This file is part of DarkFi (https://dark.fi) * * Copyright (C) 2020-2025 Dyne.org foundation * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ use std::sync::Arc; use log::{debug, error, info, warn}; use tinyjson::JsonValue; use url::Url; use darkfi::{ blockchain::BlockInfo, rpc::{ client::RpcClient, jsonrpc::{ ErrorCode::{InternalError, InvalidParams, ParseError}, JsonError, JsonRequest, JsonResponse, JsonResult, }, }, system::{Publisher, StoppableTask, StoppableTaskPtr}, util::encoding::base64, Error, Result, }; use darkfi_serial::deserialize_async; use crate::{error::handle_database_error, Explorerd}; impl Explorerd { // Queries darkfid for a block with given height. async fn get_darkfid_block_by_height(&self, height: u32) -> Result { let params = self .darkfid_daemon_request( "blockchain.get_block", &JsonValue::Array(vec![JsonValue::String(height.to_string())]), ) .await?; let param = params.get::().unwrap(); let bytes = base64::decode(param).unwrap(); let block = deserialize_async(&bytes).await?; Ok(block) } /// Synchronizes blocks between the explorer and a Darkfi blockchain node, ensuring /// the database remains consistent by syncing any missing or outdated blocks. /// /// If provided `reset` is true, the explorer's blockchain-related and metric sled trees are purged /// and syncing starts from the genesis block. The function also handles reorgs by re-aligning the /// explorer state to the correct height when blocks are outdated. Returns a result indicating /// success or failure. /// /// Reorg handling is delegated to the [`Self::process_sync_blocks_reorg`] function, whose /// documentation provides more details on the reorg process during block syncing. pub async fn sync_blocks(&self, reset: bool) -> Result<()> { // Grab last synced block height from the explorer's database. let last_synced_block = self.service.last_block().map_err(|e| { handle_database_error( "rpc_blocks::sync_blocks", "[sync_blocks] Retrieving last synced block failed", e, ) })?; // Grab the last confirmed block height and hash from the darkfi node let (last_darkfid_height, last_darkfid_hash) = self.get_last_confirmed_block().await?; // Initialize the current height to sync from, starting from genesis block if last sync block does not exist let (last_synced_height, last_synced_hash) = last_synced_block .map_or((0, "".to_string()), |(height, header_hash)| (height, header_hash)); // Declare a mutable variable to track the current sync height while processing blocks let mut current_height = last_synced_height; info!(target: "blockchain-explorer::rpc_blocks::sync_blocks", "Requested to sync from block number: {current_height}"); info!(target: "blockchain-explorer::rpc_blocks::sync_blocks", "Last confirmed block number reported by darkfid: {last_darkfid_height} - {last_darkfid_hash}"); // A reorg is detected if the hash of the last synced block differs from the hash of the last confirmed block, // unless the reset flag is set or the current height is 0 let reorg_detected = last_synced_hash != last_darkfid_hash && !reset && current_height != 0; // If the reset flag is set, reset the explorer state and start syncing from the genesis block height. // Otherwise, handle reorgs if detected, or proceed to the next block if not at the genesis height. if reset { self.service.reset_explorer_state(0)?; current_height = 0; info!(target: "blockchain-explorer::rpc_blocks::sync_blocks", "Successfully reset explorer database based on set reset parameter"); } else if reorg_detected { current_height = self.process_sync_blocks_reorg(last_synced_height, last_darkfid_height).await?; // Log only if a reorg occurred if current_height != last_synced_height { info!(target: "blockchain-explorer::rpc_blocks::sync_blocks", "Successfully completed reorg to height: {current_height}"); } // Prepare to sync the next block after reorg current_height += 1; } else if current_height != 0 { // Resume syncing from the block after the last synced height current_height += 1; } // Sync blocks until the explorer is up to date with the last confirmed block while current_height <= last_darkfid_height { // Retrieve the block from darkfi node by height let block = match self.get_darkfid_block_by_height(current_height).await { Ok(r) => r, Err(e) => { return Err(handle_database_error( "rpc_blocks::sync_blocks", "[sync_blocks] RPC client request failed", e, )) } }; // Store the retrieved block in the explorer's database if let Err(e) = self.service.put_block(&block).await { return Err(handle_database_error( "rpc_blocks::sync_blocks", "[sync_blocks] Put block failed", e, )) }; info!(target: "blockchain-explorer::rpc_blocks::sync_blocks", "Synced block {current_height}"); // Increment the current height to sync the next block current_height += 1; } info!(target: "blockchain-explorer::rpc_blocks::sync_blocks", "Completed sync, total number of explorer blocks: {}", self.service.db.blockchain.blocks.len()); Ok(()) } /// Handles blockchain reorganizations (reorgs) during the explorer node's startup synchronization /// with Darkfi nodes, ensuring the explorer provides a consistent and accurate view of the blockchain. /// /// A reorg occurs when the blocks stored by the blockchain nodes diverge from those stored by the explorer. /// This function resolves inconsistencies by identifying the point of divergence, searching backward through /// block heights, and comparing block hashes between the explorer database and the blockchain node. Once a /// common block height is found, the explorer is re-aligned to that height. /// /// If no common block can be found, the explorer resets to the "genesis height," removing all blocks, /// transactions, and metrics from its database to resynchronize with the canonical chain from the nodes. /// /// Returns the last height at which the explorer's state was successfully re-aligned with the blockchain. async fn process_sync_blocks_reorg( &self, last_synced_height: u32, last_darkfid_height: u32, ) -> Result { // Log reorg detection in the case that explorer height is greater or equal to height of darkfi node if last_synced_height >= last_darkfid_height { info!(target: "blockchain-explorer::rpc_blocks::process_sync_blocks_reorg", "Reorg detected with heights: explorer.{last_synced_height} >= darkfid.{last_darkfid_height}"); } // Declare a mutable variable to track the current height while searching for a common block let mut cur_height = last_synced_height; // Search for an explorer block that matches a darkfi node block while cur_height > 0 { let synced_block = self.service.get_block_by_height(cur_height)?; debug!(target: "blockchain-explorer::rpc_blocks::process_sync_blocks_reorg", "Searching for common block: {}", cur_height); // Check if we found a synced block for current height being searched if let Some(synced_block) = synced_block { // Fetch the block from darkfi node to check for a match match self.get_darkfid_block_by_height(cur_height).await { Ok(darkfid_block) => { // If hashes match, we've found the point of divergence if synced_block.header_hash == darkfid_block.hash().to_string() { // If hashes match but the cur_height differs from the last synced height, reset the explorer state if cur_height != last_synced_height { self.service.reset_explorer_state(cur_height)?; debug!(target: "blockchain-explorer::rpc_blocks::process_sync_blocks_reorg", "Successfully completed reorg to height: {cur_height}"); } break; } else { // Log reorg detection with height and header hash mismatch details if cur_height == last_synced_height { info!( target: "blockchain-explorer::rpc_blocks::process_sync_blocks_reorg", "Reorg detected at height {}: explorer.{} != darkfid.{}", cur_height, synced_block.header_hash, darkfid_block.hash().to_string() ); } } } // Continue searching for blocks that do not exist on darkfi nodes Err(Error::JsonRpcError((-32121, _))) => (), Err(e) => { return Err(handle_database_error( "rpc_blocks::process_sync_blocks_reorg", "[process_sync_blocks_reorg] RPC client request failed", e, )) } } } // Move to previous block to search for a match cur_height = cur_height.saturating_sub(1); } // Check if genesis block reorg is needed if cur_height == 0 { self.service.reset_explorer_state(0)?; } // Return the last height we reorged to Ok(cur_height) } // RPCAPI: // Queries the database to retrieve last N blocks. // Returns an array of readable blocks upon success. // // **Params:** // * `array[0]`: `u16` Number of blocks to retrieve (as string) // // **Returns:** // * Array of `BlockRecord` encoded into a JSON. // // --> {"jsonrpc": "2.0", "method": "blocks.get_last_n_blocks", "params": ["10"], "id": 1} // <-- {"jsonrpc": "2.0", "result": {...}, "id": 1} pub async fn blocks_get_last_n_blocks(&self, id: u16, params: JsonValue) -> JsonResult { let params = params.get::>().unwrap(); if params.len() != 1 || !params[0].is_string() { return JsonError::new(InvalidParams, None, id).into() } // Extract the number of last blocks to retrieve from parameters let n = match params[0].get::().unwrap().parse::() { Ok(v) => v, Err(_) => return JsonError::new(ParseError, None, id).into(), }; // Fetch the blocks and handle potential errors let blocks_result = match self.service.get_last_n(n) { Ok(blocks) => blocks, Err(e) => { error!(target: "blockchain-explorer::rpc_blocks::blocks_get_last_n_blocks", "Failed fetching blocks: {}", e); return JsonError::new(InternalError, None, id).into(); } }; // Transform blocks to json and return result if blocks_result.is_empty() { JsonResponse::new(JsonValue::Array(vec![]), id).into() } else { let json_blocks: Vec = blocks_result.into_iter().map(|block| block.to_json_array()).collect(); JsonResponse::new(JsonValue::Array(json_blocks), id).into() } } // RPCAPI: // Queries the database to retrieve blocks in provided heights range. // Returns an array of readable blocks upon success. // // **Params:** // * `array[0]`: `u32` Starting height (as string) // * `array[1]`: `u32` Ending height range (as string) // // **Returns:** // * Array of `BlockRecord` encoded into a JSON. // // --> {"jsonrpc": "2.0", "method": "blocks.get_blocks_in_heights_range", "params": ["10", "15"], "id": 1} // <-- {"jsonrpc": "2.0", "result": {...}, "id": 1} pub async fn blocks_get_blocks_in_heights_range( &self, id: u16, params: JsonValue, ) -> JsonResult { let params = params.get::>().unwrap(); if params.len() != 2 || !params[0].is_string() || !params[1].is_string() { return JsonError::new(InvalidParams, None, id).into() } let start = match params[0].get::().unwrap().parse::() { Ok(v) => v, Err(_) => return JsonError::new(ParseError, None, id).into(), }; let end = match params[1].get::().unwrap().parse::() { Ok(v) => v, Err(_) => return JsonError::new(ParseError, None, id).into(), }; if start > end { return JsonError::new(ParseError, None, id).into() } // Fetch the blocks and handle potential errors let blocks_result = match self.service.get_by_range(start, end) { Ok(blocks) => blocks, Err(e) => { error!(target: "blockchain-explorer::rpc_blocks::blocks_get_blocks_in_height_range", "Failed fetching blocks: {}", e); return JsonError::new(InternalError, None, id).into(); } }; // Transform blocks to json and return result if blocks_result.is_empty() { JsonResponse::new(JsonValue::Array(vec![]), id).into() } else { let json_blocks: Vec = blocks_result.into_iter().map(|block| block.to_json_array()).collect(); JsonResponse::new(JsonValue::Array(json_blocks), id).into() } } // RPCAPI: // Queries the database to retrieve the block corresponding to the provided hash. // Returns the readable block upon success. // // **Params:** // * `array[0]`: `String` Block header hash // // **Returns:** // * `BlockRecord` encoded into a JSON. // // --> {"jsonrpc": "2.0", "method": "blocks.get_block_by_hash", "params": ["5cc...2f9"], "id": 1} // <-- {"jsonrpc": "2.0", "result": {...}, "id": 1} pub async fn blocks_get_block_by_hash(&self, id: u16, params: JsonValue) -> JsonResult { let params = params.get::>().unwrap(); if params.len() != 1 || !params[0].is_string() { return JsonError::new(InvalidParams, None, id).into() } // Extract header hash from params, returning error if not provided let header_hash = match params[0].get::() { Some(hash) => hash, None => return JsonError::new(InvalidParams, None, id).into(), }; // Fetch and transform block to json, handling any errors and returning the result match self.service.get_block_by_hash(header_hash) { Ok(Some(block)) => JsonResponse::new(block.to_json_array(), id).into(), Ok(None) => JsonResponse::new(JsonValue::Array(vec![]), id).into(), Err(e) => { error!(target: "blockchain-explorer::rpc_blocks", "Failed fetching block: {:?}", e); JsonError::new(InternalError, None, id).into() } } } // Queries darkfid for last confirmed block. async fn get_last_confirmed_block(&self) -> Result<(u32, String)> { let rep = self .darkfid_daemon_request("blockchain.last_confirmed_block", &JsonValue::Array(vec![])) .await?; let params = rep.get::>().unwrap(); let height = *params[0].get::().unwrap() as u32; let hash = params[1].get::().unwrap().clone(); Ok((height, hash)) } } /// Subscribes to darkfid's JSON-RPC notification endpoint that serves /// new confirmed blocks. Upon receiving them, store them to the database. pub async fn subscribe_blocks( explorer: Arc, endpoint: Url, ex: Arc>, ) -> Result<(StoppableTaskPtr, StoppableTaskPtr)> { // Grab last confirmed block let (last_darkfid_height, last_darkfid_hash) = explorer.get_last_confirmed_block().await?; // Grab last synced block let (mut height, hash) = match explorer.service.last_block() { Ok(Some((height, hash))) => (height, hash), Ok(None) => (0, "".to_string()), Err(e) => { return Err(Error::DatabaseError(format!( "[subscribe_blocks] Retrieving last synced block failed: {e:?}" ))) } }; // Evaluates whether there is a mismatch between the last confirmed block and the last synced block let blocks_mismatch = (last_darkfid_height != height || last_darkfid_hash != hash) && last_darkfid_height != 0 && height != 0; // Check if there is a mismatch, throwing an error to prevent operating in a potentially inconsistent state if blocks_mismatch { warn!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "Warning: Last synced block is not the last confirmed block: \ last_darkfid_height={last_darkfid_height}, last_synced_height={height}, last_darkfid_hash={last_darkfid_hash}, last_synced_hash={hash}"); warn!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "You should first fully sync the blockchain, and then subscribe"); return Err(Error::DatabaseError( "[subscribe_blocks] Blockchain not fully synced".to_string(), )); } info!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "Subscribing to receive notifications of incoming blocks"); let publisher = Publisher::new(); let subscription = publisher.clone().subscribe().await; let _ex = ex.clone(); let subscriber_task = StoppableTask::new(); subscriber_task.clone().start( // Weird hack to prevent lifetimes hell async move { let ex = _ex.clone(); let rpc_client = RpcClient::new(endpoint, ex).await?; let req = JsonRequest::new("blockchain.subscribe_blocks", JsonValue::Array(vec![])); rpc_client.subscribe(req, publisher).await }, |res| async move { match res { Ok(()) => { /* Do nothing */ } Err(e) => error!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "[subscribe_blocks] JSON-RPC server error: {e:?}"), } }, Error::RpcServerStopped, ex.clone(), ); info!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "Detached subscription to background"); info!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "All is good. Waiting for block notifications..."); let listener_task = StoppableTask::new(); listener_task.clone().start( // Weird hack to prevent lifetimes hell async move { loop { match subscription.receive().await { JsonResult::Notification(n) => { debug!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "Got Block notification from darkfid subscription"); if n.method != "blockchain.subscribe_blocks" { return Err(Error::UnexpectedJsonRpc(format!( "Got foreign notification from darkfid: {}", n.method ))) } // Verify parameters if !n.params.is_array() { return Err(Error::UnexpectedJsonRpc( "Received notification params are not an array".to_string(), )) } let params = n.params.get::>().unwrap(); if params.is_empty() { return Err(Error::UnexpectedJsonRpc( "Notification parameters are empty".to_string(), )) } for param in params { let param = param.get::().unwrap(); let bytes = base64::decode(param).unwrap(); let darkfid_block: BlockInfo = match deserialize_async(&bytes).await { Ok(b) => b, Err(e) => { return Err(Error::UnexpectedJsonRpc(format!( "[subscribe_blocks] Deserializing block failed: {e:?}" ))) }, }; info!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "======================================="); info!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "Block Notification: {}", darkfid_block.hash().to_string()); info!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "======================================="); // Store darkfi node block height for later use let darkfid_block_height = darkfid_block.header.height; // Check if we need to perform a reorg due to mismatch in block heights if darkfid_block_height <= height { info!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "Reorg detected with heights: darkfid.{darkfid_block_height} <= explorer.{height}"); // Calculate the reset height let reset_height = darkfid_block_height.saturating_sub(1); // Execute the reorg by resetting the explorer state to reset height explorer.service.reset_explorer_state(reset_height)?; info!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "Successfully completed reorg to height: {reset_height}"); } if let Err(e) = explorer.service.put_block(&darkfid_block).await { return Err(Error::DatabaseError(format!( "[subscribe_blocks] Put block failed: {e:?}" ))) } info!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "Successfully stored new block at height: {}", darkfid_block.header.height ); // Process the next block height = darkfid_block.header.height; } } JsonResult::Error(e) => { // Some error happened in the transmission return Err(Error::UnexpectedJsonRpc(format!("Got error from JSON-RPC: {e:?}"))) } x => { // And this is weird return Err(Error::UnexpectedJsonRpc(format!( "Got unexpected data from JSON-RPC: {x:?}" ))) } } }; }, |res| async move { match res { Ok(()) => { /* Do nothing */ } Err(e) => error!(target: "blockchain-explorer::rpc_blocks::subscribe_blocks", "[subscribe_blocks] JSON-RPC server error: {e:?}"), } }, Error::RpcServerStopped, ex, ); Ok((subscriber_task, listener_task)) }