/* This file is part of DarkFi (https://dark.fi)
*
* Copyright (C) 2020-2026 Dyne.org foundation
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
use std::{
collections::HashSet,
path::{Path, PathBuf},
process::exit,
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
};
use arg::Args;
use async_trait::async_trait;
use darkfi::{
blockchain::BlockInfo,
rpc::{
client::RpcClient,
jsonrpc::{ErrorCode, JsonError, JsonRequest, JsonResult},
server::{listen_and_serve, RequestHandler},
settings::RpcSettings,
},
system::{msleep, CondVar, Publisher, PublisherPtr, StoppableTask, StoppableTaskPtr},
util::{encoding::base64, path::expand_path},
verbose, Error, Result, ANSI_LOGO,
};
use darkfi_serial::deserialize_async;
use smol::{
future,
lock::{Mutex, MutexGuard},
Executor,
};
use tapes::{TapeOpenOptions, Tapes};
use tinyjson::JsonValue;
use tracing::{debug, error, info, warn};
use url::Url;
/// Database interfaces
mod db;
use db::{DifficultyIndex, TapesDatabase};
/// JSON-RPC server methods
mod rpc;
const ABOUT: &str =
concat!("explorer ", env!("CARGO_PKG_VERSION"), '\n', env!("CARGO_PKG_DESCRIPTION"));
const USAGE: &str = r#"
Usage: explorer [OPTIONS]
Options:
-e darkfid JSON-RPC endpoint (default: tcp://127.0.0.1:18345)
-d Path to database (default: ~/.local/share/darkfi/explorer/db)
-r Revert database to
-h Show this help
"#;
fn usage() {
print!("{ANSI_LOGO}{ABOUT}\n{USAGE}");
}
pub struct Explorer {
synced: AtomicBool,
synced_notifier: Arc,
_sled_db: sled::Db,
header_indices: sled::Tree,
tx_indices: sled::Tree,
contracts: sled::Tree,
stats: sled::Tree,
tapes_db: Tapes,
_tapes_options: TapeOpenOptions,
database: TapesDatabase,
rpc_sub: StoppableTaskPtr,
rpc_sub_handler: StoppableTaskPtr,
blocks_publisher: PublisherPtr,
rpc_connections: Mutex>,
}
struct RpcHandler;
#[async_trait]
impl RequestHandler for Explorer {
async fn handle_request(&self, req: JsonRequest) -> JsonResult {
debug!(target: "explorer::rpc", "--> {}", req.stringify().unwrap());
match req.method.as_str() {
"current_difficulty" => self.rpc_current_difficulty(req.id, req.params).await,
"current_height" => self.rpc_current_height(req.id, req.params).await,
"latest_blocks" => self.rpc_latest_blocks(req.id, req.params).await,
"get_block" => self.rpc_get_block(req.id, req.params).await,
"get_tx" => self.rpc_get_tx(req.id, req.params).await,
"search" => self.rpc_search(req.id, req.params).await,
"get_hashrate" => self.rpc_get_hashrate(req.id, req.params).await,
"get_contract" => self.rpc_get_contract(req.id, req.params).await,
"list_contracts" => self.rpc_list_contracts(req.id, req.params).await,
"contract_count" => self.rpc_contract_count(req.id, req.params).await,
"get_stats" => self.rpc_get_stats(req.id, req.params).await,
_ => JsonError::new(ErrorCode::MethodNotFound, None, req.id).into(),
}
}
async fn connections_mut(&self) -> MutexGuard<'life0, HashSet> {
self.rpc_connections.lock().await
}
}
impl Explorer {
fn new(sled_path: &Path, tapes_db_path: &Path, tapes_path: &Path) -> Result {
info!(target: "explorer::new", "Opening sled trees");
let sled_db = sled::open(sled_path)?;
let header_indices = sled_db.open_tree("header_indices")?;
let tx_indices = sled_db.open_tree("tx_indices")?;
let contracts = sled_db.open_tree("contracts")?;
let stats = sled_db.open_tree("stats")?;
info!(target: "explorer::new", "Opening tapes");
std::fs::create_dir_all(tapes_db_path)?;
std::fs::create_dir_all(tapes_path)?;
let tapes_db = Tapes::open(tapes_db_path)?;
let tapes_options =
TapeOpenOptions { top_cache_size: 64 * 1024, dir: tapes_path.to_path_buf() };
let database = Self::open_tapes(&tapes_db, &tapes_options)?;
Ok(Self {
synced: AtomicBool::new(false),
synced_notifier: Arc::new(CondVar::new()),
_sled_db: sled_db,
header_indices,
tx_indices,
contracts,
stats,
tapes_db,
_tapes_options: tapes_options,
database,
rpc_sub: StoppableTask::new(),
rpc_sub_handler: StoppableTask::new(),
blocks_publisher: Publisher::new(),
rpc_connections: Mutex::new(HashSet::new()),
})
}
async fn handle_block_sub(&self, rpc_endpoint: Url, ex: Arc>) -> Result<()> {
info!(
target: "explorer::handle_block_sub",
"Started block subscription, waiting until blockchain is synced",
);
let block_subscription = self.blocks_publisher.clone().subscribe().await;
self.synced_notifier.wait().await;
info!(
target: "explorer::handle_block_sub",
"Blockchain synced, now waiting for new blocks...",
);
loop {
// Handle the new block. We get a JsonResult, so also handle
// any errors that might arise.
let block_notification = block_subscription.receive().await;
info!(target: "explorer::handle_block_sub", "Got new block notification!");
match block_notification {
JsonResult::Notification(notification) => {
for param in notification.params.get::>().unwrap() {
// Deserialize base64 block
let block_bytes = base64::decode(param.get::().unwrap()).unwrap();
let block: BlockInfo = deserialize_async(&block_bytes).await.unwrap();
let incoming_height = block.header.height as u64;
info!(target: "explorer::handle_block_sub", "Height {}", incoming_height);
// Check if we need to reorg or sync
let current_height = self.get_height().ok().flatten().unwrap_or(0);
if incoming_height > current_height + 1 {
// Sync needed: we have at least one missing block
info!(
target: "explorer::handle_block_sub",
"Sync needed! Incoming height {} > current_height {}.",
incoming_height, current_height,
);
self.synced.store(false, Ordering::SeqCst);
self.sync_blockchain(
rpc_endpoint.clone(),
current_height + 1,
incoming_height - 1,
ex.clone(),
)
.await?;
self.synced.store(true, Ordering::SeqCst);
info!(
target: "explorer::handle_block_sub",
"Synced to height {}", incoming_height - 1,
);
}
if incoming_height <= current_height {
// Reorg needed: incoming block is at or before our current height
let blocks_to_revert = current_height - incoming_height + 1;
info!(
target: "explorer::handle_block_sub",
"Reorg detected! Incoming height {} <= current height {}. Reverting {} blocks.",
incoming_height, current_height, blocks_to_revert
);
if let Err(e) = self.revert_to_height(incoming_height - 1).await {
error!(
target: "explorer::handle_block_sub",
"Failed to revert blocks during reorg: {e}",
);
// Exit from this task if there's an error.
// It'll let us inspect the db and what happened.
return Err(e.into())
}
}
// Get difficulty
let rpc_client =
RpcClient::new(rpc_endpoint.clone(), ex.clone()).await.unwrap();
let req = JsonRequest::new(
"blockchain.get_difficulty",
JsonValue::Array(vec![(block.header.height as f64).into()]),
);
let rep = rpc_client.request(req).await?;
rpc_client.stop().await;
let params = rep.get::>().unwrap();
let difficulty = *params[0].get::().unwrap() as u64;
let cumulative = *params[1].get::().unwrap() as u64;
let diff = DifficultyIndex { difficulty, cumulative };
self.append_block(&block, &diff).await.unwrap();
}
}
x => unreachable!("{:?}", x),
}
}
}
async fn sync_blockchain(
&self,
rpc_endpoint: Url,
from_height: u64,
to_height: u64,
ex: Arc>,
) -> Result<()> {
if from_height >= to_height {
return Ok(())
}
info!(
target: "explorer::sync_blockchain",
"Started blockchain sync from_height={from_height} to_height={to_height}...",
);
let rpc_client = Arc::new(RpcClient::new(rpc_endpoint, ex.clone()).await?);
for height in from_height..=to_height {
info!(target: "explorer::sync_blockchain", "Requesting block at height {height}");
// Get block
let req = JsonRequest::new(
"blockchain.get_block",
JsonValue::Array(vec![(height as f64).into()]),
);
let rep = rpc_client.request(req).await?;
let param = rep.get::().unwrap();
let bytes = base64::decode(param).unwrap();
let block: BlockInfo = deserialize_async(&bytes).await?;
// Get difficulty
let req = JsonRequest::new(
"blockchain.get_difficulty",
JsonValue::Array(vec![(height as f64).into()]),
);
let rep = rpc_client.request(req).await?;
let params = rep.get::>().unwrap();
let difficulty = *params[0].get::().unwrap() as u64;
let cumulative = *params[1].get::().unwrap() as u64;
let diff = DifficultyIndex { difficulty, cumulative };
self.append_block(&block, &diff).await?;
}
rpc_client.stop().await;
Ok(())
}
}
async fn realmain(
rpc_endpoint: Url,
db_path: PathBuf,
revert_to: u64,
ex: Arc>,
) -> Result<()> {
let explorer = Arc::new(Explorer::new(
&db_path.join("sled_db"),
&db_path.join("tapes_metadata"),
&db_path.join("tapes"),
)?);
// First we should subscribe to new blocks and queue them to apply
// after we sync. For this we create a new longterm background task
// that will handle incoming blocks. It will wait until the blockchain
// is synced and then proceed to process them.
let explorer_ = Arc::clone(&explorer);
let ex_ = ex.clone();
let rpc_endpoint_ = rpc_endpoint.clone();
explorer.rpc_sub_handler.clone().start(
async move { explorer_.handle_block_sub(rpc_endpoint_, ex_).await },
|_| async {},
Error::RpcServerStopped,
ex.clone(),
);
// Then we subscribe to darkfid's RPC to get new blocks. We should first
// fetch the current height, so we know how far to sync. Then any blocks
// that come after that should be queued in the `blocks_publisher`.
info!(target: "explorer", "Connecting to darkfid RPC...");
let rpc_client = loop {
let Ok(rpc_client) = RpcClient::new(rpc_endpoint.clone(), ex.clone()).await else {
msleep(500).await;
continue
};
break rpc_client
};
let req = JsonRequest::new("blockchain.last_confirmed_block", JsonValue::Array(vec![]));
let rep = rpc_client.request(req).await?;
rpc_client.stop().await;
let params = rep.get::>().unwrap();
let confirmed_height = *params[0].get::().unwrap() as u64;
// Now create the subscription task.
let explorer_ = Arc::clone(&explorer);
let ex_ = Arc::clone(&ex);
let rpc_endpoint_ = rpc_endpoint.clone();
explorer.rpc_sub.clone().start(
async move {
loop {
let rpc_client = match RpcClient::new(rpc_endpoint_.clone(), ex_.clone()).await {
Ok(v) => v,
Err(e) => {
warn!(target: "explorer::subscribe_blocks", "darkfid RPC connection lost ({e})), retrying...");
msleep(500).await;
continue
}
};
info!(target: "explorer::subscribe_blocks", "Connected to darkfid RPC");
let req = JsonRequest::new("blockchain.subscribe_blocks", JsonValue::Array(vec![]));
if let Err(e) = rpc_client.subscribe(req, explorer_.blocks_publisher.clone()).await {
rpc_client.stop().await;
warn!(target: "explorer::subscribe_blocks", "darkfid RPC connection lost ({e}), retrying...");
msleep(500).await;
}
}
},
|_| async {},
Error::RpcServerStopped,
ex.clone(),
);
// Once the tasks are set up, we'll now perform a manual sync up to
// the last confirmed height. This will create a new RPC client that
// is going to request and parse all the necessary blocks, and then
// apply them to the databases.
let mut sync_from = explorer.get_height()?.unwrap_or(0);
if sync_from > 0 {
// If we're not syncing from genesis, account for it.
sync_from += 1;
}
explorer.sync_blockchain(rpc_endpoint.clone(), sync_from, confirmed_height, ex.clone()).await?;
explorer.synced.store(true, Ordering::SeqCst);
explorer.synced_notifier.notify();
if revert_to > 0 {
info!("Reverting to {}", revert_to);
explorer.revert_to_height(revert_to).await?;
}
// Start up an RPC server that can be queried for data.
// This normally serves the data to the Python website frontend.
info!(target: "explorer", "Starting JSONRPC server");
let rpc_settings = RpcSettings::default();
listen_and_serve(rpc_settings, explorer, None, ex.clone()).await?;
Ok(())
}
fn main() -> Result<()> {
let mut hflag = false;
let mut evalue = "tcp://127.0.0.1:18345".to_string();
let mut dvalue = "~/.local/share/darkfi/explorer/db".to_string();
let mut rvalue = "0".to_string();
let mut verbose = 0;
{
let mut args = Args::new().with_cb(|args, flag| match flag {
'e' => evalue = args.eargf().to_string(),
'd' => dvalue = args.eargf().to_string(),
'r' => rvalue = args.eargf().to_string(),
'v' => verbose += 1,
_ => hflag = true,
});
args.parse();
}
let revert_to: u64 = rvalue.parse()?;
if hflag {
usage();
exit(1);
}
let rpc_endpoint: Url = match evalue.parse() {
Ok(v) => v,
Err(e) => {
println!("Error parsing RPC endpoint: {e}");
usage();
exit(1);
}
};
let db_path: PathBuf = match expand_path(&dvalue) {
Ok(v) => v,
Err(e) => {
println!("Error parsing DB path: {e}");
usage();
exit(1);
}
};
let ex = Arc::new(Executor::new());
let (signal, shutdown) = async_channel::unbounded::<()>();
darkfi::util::logger::setup_logging(verbose, None)?;
info!(target: "explorer", "RPC Endpoint: {}", evalue);
info!(target: "explorer", "DB Path: {}", dvalue);
verbose!(target: "explorer", "Log Level: {}", verbose);
let (_, result) = easy_parallel::Parallel::new()
// Run four executor threads
.each(0..4, |_| future::block_on(ex.run(shutdown.recv())))
// Run the main future on the current thread
.finish(|| {
future::block_on(async {
realmain(rpc_endpoint, db_path, revert_to, ex.clone()).await?;
drop(signal);
Ok::<(), darkfi::Error>(())
})
});
result
}