/* This file is part of DarkFi (https://dark.fi) * * Copyright (C) 2020-2026 Dyne.org foundation * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ // cargo test --release --features=event-graph --lib eventgraph_propagation -- --include-ignored use std::{collections::HashMap, slice, sync::Arc}; use rand::{prelude::SliceRandom, rngs::ThreadRng}; use sled_overlay::sled; use smol::{channel, future, Executor}; use tracing::{info, warn}; use url::Url; use crate::{ event_graph::{ proto::{EventPut, ProtocolEventGraph}, Event, EventGraph, }, net::{session::SESSION_DEFAULT, settings::NetworkProfile, P2p, Settings}, system::{msleep, sleep}, util::logger::{setup_test_logger, Level}, }; // Number of nodes to spawn and number of peers each node connects to const N_NODES: usize = 5; const N_CONNS: usize = 2; //const N_NODES: usize = 50; //const N_CONNS: usize = N_NODES / 3; fn init_logger() { let ignored_targets = [ "sled", "net::protocol_ping", "net::channel::subscribe_stop()", "net::hosts", "net::session", "net::message_subscriber", "net::protocol_address", "net::protocol_version", "net::protocol_registry", "net::channel::send()", "net::channel::start()", "net::channel::subscribe_msg()", "net::channel::main_receive_loop()", "net::tcp", ]; // We check this error so we can execute same file tests in parallel, // otherwise second one fails to init logger here. if setup_test_logger( &ignored_targets, false, Level::Info, //Level::Verbose, //Level::Debug, //Level::Tracing, ) .is_err() { warn!(target: "test_harness", "Logger already initialized"); } } async fn spawn_node( inbound_addrs: Vec, peers: Vec, ex: Arc>, ) -> Arc { let mut profiles = HashMap::new(); profiles.insert( "tcp".to_string(), NetworkProfile { outbound_connect_timeout: 2, ..Default::default() }, ); let settings = Settings { localnet: true, inbound_addrs, outbound_connections: 0, inbound_connections: usize::MAX, peers, active_profiles: vec!["tcp".to_string()], profiles, ..Default::default() }; let p2p = P2p::new(settings, ex.clone()).await.unwrap(); let sled_db = sled::Config::new().temporary(true).open().unwrap(); let event_graph = EventGraph::new(p2p.clone(), sled_db, "/tmp".into(), false, false, 1, ex.clone()) .await .unwrap(); *event_graph.synced.write().await = true; let event_graph_ = event_graph.clone(); // Register the P2P protocols let registry = p2p.protocol_registry(); registry .register(SESSION_DEFAULT, move |channel, _| { let event_graph_ = event_graph_.clone(); async move { ProtocolEventGraph::init(event_graph_, channel).await.unwrap() } }) .await; event_graph } async fn bootstrap_nodes( peer_indexes: &[usize], starting_port: usize, rng: &mut ThreadRng, ex: Arc>, ) -> Vec> { let mut eg_instances = vec![]; // Initialize the nodes for i in 0..N_NODES { // Everyone will connect to N_CONNS random peers. let mut peer_indexes_copy = peer_indexes.to_owned(); peer_indexes_copy.remove(i); let peer_indexes_to_connect: Vec<_> = peer_indexes_copy.choose_multiple(rng, N_CONNS).collect(); let mut peers = vec![]; for peer_index in peer_indexes_to_connect { let port = starting_port + peer_index; peers.push(Url::parse(&format!("tcp://127.0.0.1:{port}")).unwrap()); } let event_graph = spawn_node( vec![Url::parse(&format!("tcp://127.0.0.1:{}", starting_port + i)).unwrap()], peers, ex.clone(), ) .await; eg_instances.push(event_graph); } // Start the P2P network for eg in eg_instances.iter() { eg.p2p.clone().start().await.unwrap(); } info!("Waiting 5s until all peers connect"); sleep(5).await; eg_instances } async fn assert_dags(eg_instances: &[Arc], expected_len: usize, rng: &mut ThreadRng) { let random_node = eg_instances.choose(rng).unwrap(); let random_node_genesis = random_node.current_genesis.read().await.id(); let store = random_node.dag_store.read().await; let (_, unreferenced_tips) = store.main_dags.get(&random_node_genesis).unwrap(); let last_layer_tips = unreferenced_tips.last_key_value().unwrap().1.clone(); for (i, eg) in eg_instances.iter().enumerate() { let current_genesis = eg.current_genesis.read().await; let dag_name = current_genesis.id().to_string(); let dag = eg.dag_store.read().await.get_dag(&dag_name); let unreferenced_tips = eg.dag_store.read().await.find_unreferenced_tips(&dag).await; let node_last_layer_tips = unreferenced_tips.last_key_value().unwrap().1.clone(); assert!( dag.len() == expected_len, "Node {i}, expected {expected_len} events, have {}", dag.len() ); assert_eq!( node_last_layer_tips, last_layer_tips, "Node {i} contains malformed unreferenced tips" ); } } macro_rules! test_body { ($real_call:ident) => { init_logger(); let ex = Arc::new(Executor::new()); let ex_ = ex.clone(); let (signal, shutdown) = channel::unbounded::<()>(); // Run a thread for each node. easy_parallel::Parallel::new() .each(0..N_NODES, |_| future::block_on(ex.run(shutdown.recv()))) .finish(|| { future::block_on(async { $real_call(ex_).await; drop(signal); }) }); }; } #[test] fn eventgraph_propagation() { test_body!(eventgraph_propagation_real); } async fn eventgraph_propagation_real(ex: Arc>) { let mut rng = rand::thread_rng(); let peer_indexes: Vec = (0..N_NODES).collect(); // Bootstrap nodes let mut eg_instances = bootstrap_nodes(&peer_indexes, 13200, &mut rng, ex.clone()).await; // Grab genesis event let random_node = eg_instances.choose(&mut rng).unwrap(); let current_genesis = random_node.current_genesis.read().await; let dag_name = current_genesis.id().to_string(); let (id, _) = random_node.dag_store.read().await.get_dag(&dag_name).last().unwrap().unwrap(); let genesis_event_id = blake3::Hash::from_bytes((&id as &[u8]).try_into().unwrap()); drop(current_genesis); // ========================================= // 1. Assert that everyone's DAG is the same // ========================================= assert_dags(&eg_instances, 1, &mut rng).await; // ========================================== // 2. Create an event in one node and publish // ========================================== let random_node = eg_instances.choose(&mut rng).unwrap(); let current_genesis = random_node.current_genesis.read().await; let dag_name = current_genesis.id().to_string(); let event = Event::new(vec![1, 2, 3, 4], random_node).await; assert!(event.header.parents.contains(&genesis_event_id)); // The node adds it to their DAG, on layer 1. random_node.header_dag_insert(vec![event.header.clone()], &dag_name).await.unwrap(); let event_id = random_node.dag_insert(slice::from_ref(&event), &dag_name).await.unwrap()[0]; let store = random_node.dag_store.read().await; let (_, tips_layers) = store.header_dags.get(¤t_genesis.id()).unwrap(); // Since genesis was referenced, its layer (0) have been removed assert_eq!(tips_layers.len(), 1); assert!(tips_layers.last_key_value().unwrap().1.get(&event_id).is_some()); drop(store); drop(current_genesis); info!("Broadcasting event {event_id}"); random_node.p2p.broadcast(&EventPut(event)).await; info!("Waiting 5s for event propagation"); sleep(5).await; // ==================================================== // 3. Assert that everyone has the new event in the DAG // ==================================================== assert_dags(&eg_instances, 2, &mut rng).await; // ============================================================== // 4. Create multiple events on a node and broadcast the last one // The `EventPut` logic should manage to fetch all of them, // provided that the last one references the earlier ones. // ============================================================== let random_node = eg_instances.choose(&mut rng).unwrap(); let event0 = Event::new(vec![1, 2, 3, 4, 0], random_node).await; random_node.header_dag_insert(vec![event0.header.clone()], &dag_name).await.unwrap(); let event0_id = random_node.dag_insert(slice::from_ref(&event0), &dag_name).await.unwrap()[0]; let event1 = Event::new(vec![1, 2, 3, 4, 1], random_node).await; random_node.header_dag_insert(vec![event1.header.clone()], &dag_name).await.unwrap(); let event1_id = random_node.dag_insert(slice::from_ref(&event1), &dag_name).await.unwrap()[0]; let event2 = Event::new(vec![1, 2, 3, 4, 2], random_node).await; random_node.header_dag_insert(vec![event2.header.clone()], &dag_name).await.unwrap(); let event2_id = random_node.dag_insert(slice::from_ref(&event2), &dag_name).await.unwrap()[0]; // Genesis event + event from 2. + upper 3 events (layer 4) let current_genesis = random_node.current_genesis.read().await; let dag_name = current_genesis.id().to_string(); assert_eq!(random_node.dag_store.read().await.get_dag(&dag_name).len(), 5); let random_node_genesis = random_node.current_genesis.read().await.id(); let store = random_node.dag_store.read().await; let (_, tips_layers) = store.header_dags.get(&random_node_genesis).unwrap(); assert_eq!(tips_layers.len(), 1); assert!(tips_layers.get(&4).unwrap().get(&event2_id).is_some()); drop(current_genesis); drop(store); let event_chain = vec![ (event0_id, event0.header.parents), (event1_id, event1.header.parents), (event2_id, event2.header.parents), ]; info!("Broadcasting event {event2_id}"); info!("Event chain: {event_chain:#?}"); random_node.p2p.broadcast(&EventPut(event2)).await; info!("Waiting 5s for event propagation"); sleep(5).await; // ========================================== // 5. Assert that everyone has all the events // ========================================== assert_dags(&eg_instances, 5, &mut rng).await; // =========================================== // 6. Create multiple events on multiple nodes // =========================================== // node 1 // ======= let node1 = eg_instances.choose(&mut rng).unwrap(); let event0_1 = Event::new(vec![1, 2, 3, 4, 3], node1).await; node1.header_dag_insert(vec![event0_1.header.clone()], &dag_name).await.unwrap(); node1.dag_insert(slice::from_ref(&event0_1), &dag_name).await.unwrap(); node1.p2p.broadcast(&EventPut(event0_1)).await; msleep(300).await; let event1_1 = Event::new(vec![1, 2, 3, 4, 4], node1).await; node1.header_dag_insert(vec![event1_1.header.clone()], &dag_name).await.unwrap(); node1.dag_insert(slice::from_ref(&event1_1), &dag_name).await.unwrap(); node1.p2p.broadcast(&EventPut(event1_1)).await; msleep(300).await; let event2_1 = Event::new(vec![1, 2, 3, 4, 5], node1).await; node1.header_dag_insert(vec![event2_1.header.clone()], &dag_name).await.unwrap(); node1.dag_insert(slice::from_ref(&event2_1), &dag_name).await.unwrap(); node1.p2p.broadcast(&EventPut(event2_1)).await; msleep(300).await; // ======= // node 2 // ======= let node2 = eg_instances.choose(&mut rng).unwrap(); let event0_2 = Event::new(vec![1, 2, 3, 4, 6], node2).await; node2.header_dag_insert(vec![event0_2.header.clone()], &dag_name).await.unwrap(); node2.dag_insert(slice::from_ref(&event0_2), &dag_name).await.unwrap(); node2.p2p.broadcast(&EventPut(event0_2)).await; msleep(300).await; let event1_2 = Event::new(vec![1, 2, 3, 4, 7], node2).await; node2.header_dag_insert(vec![event1_2.header.clone()], &dag_name).await.unwrap(); node2.dag_insert(slice::from_ref(&event1_2), &dag_name).await.unwrap(); node2.p2p.broadcast(&EventPut(event1_2)).await; msleep(300).await; let event2_2 = Event::new(vec![1, 2, 3, 4, 8], node2).await; node2.header_dag_insert(vec![event2_2.header.clone()], &dag_name).await.unwrap(); node2.dag_insert(slice::from_ref(&event2_2), &dag_name).await.unwrap(); node2.p2p.broadcast(&EventPut(event2_2)).await; msleep(300).await; // ======= // node 3 // ======= let node3 = eg_instances.choose(&mut rng).unwrap(); let event0_3 = Event::new(vec![1, 2, 3, 4, 9], node3).await; node3.header_dag_insert(vec![event0_3.header.clone()], &dag_name).await.unwrap(); node3.dag_insert(slice::from_ref(&event0_3), &dag_name).await.unwrap(); node3.p2p.broadcast(&EventPut(event0_3)).await; msleep(300).await; let event1_3 = Event::new(vec![1, 2, 3, 4, 10], node3).await; node3.header_dag_insert(vec![event1_3.header.clone()], &dag_name).await.unwrap(); node3.dag_insert(slice::from_ref(&event1_3), &dag_name).await.unwrap(); node3.p2p.broadcast(&EventPut(event1_3)).await; msleep(300).await; let event2_3 = Event::new(vec![1, 2, 3, 4, 11], node3).await; node3.header_dag_insert(vec![event2_3.header.clone()], &dag_name).await.unwrap(); node3.dag_insert(slice::from_ref(&event2_3), &dag_name).await.unwrap(); node3.p2p.broadcast(&EventPut(event2_3)).await; msleep(300).await; // ///// // // // let node4 = eg_instances.choose(&mut rng).unwrap(); // let event0_4 = Event::new(vec![1, 2, 3, 4, 12], node4).await; // node4.dag_insert(&[event0_4.clone()]).await.unwrap(); // node4.p2p.broadcast(&EventPut(event0_4)).await; // sleep(1).await; // let event1_4 = Event::new(vec![1, 2, 3, 4, 13], node4).await; // node4.dag_insert(&[event1_4.clone()]).await.unwrap(); // node4.p2p.broadcast(&EventPut(event1_4)).await; // sleep(1).await; // let event2_4 = Event::new(vec![1, 2, 3, 4, 14], node4).await; // node4.dag_insert(&[event2_4.clone()]).await.unwrap(); // node4.p2p.broadcast(&EventPut(event2_4)).await; // // sleep(1).await; // ========================================== // 7. Assert that everyone has all the events // ========================================== // 5 events from 2. and 4. + 9 events from 6. = 14 assert_dags(&eg_instances, 14, &mut rng).await; // ============================================================ // 8. Start a new node and try to sync the DAG from other peers // ============================================================ { // Connect to N_CONNS random peers. let peer_indexes_to_connect: Vec<_> = peer_indexes.choose_multiple(&mut rng, N_CONNS).collect(); let mut peers = vec![]; for peer_index in peer_indexes_to_connect { let port = 13200 + peer_index; peers.push(Url::parse(&format!("tcp://127.0.0.1:{port}")).unwrap()); } let event_graph = spawn_node( vec![Url::parse(&format!("tcp://127.0.0.1:{}", 13200 + N_NODES + 1)).unwrap()], peers, ex.clone(), ) .await; eg_instances.push(event_graph.clone()); event_graph.p2p.clone().start().await.unwrap(); info!("Waiting 5s for new node connection"); sleep(5).await; event_graph.sync_selected(1, false).await.unwrap(); } // ============================================================ // 9. Assert the new synced DAG has the same contents as others // ============================================================ // 5 events from 2. and 4. + 9 events from 6. = 14 assert_dags(&eg_instances, 14, &mut rng).await; // Stop the P2P network for eg in eg_instances.iter() { eg.p2p.clone().stop().await; } } #[test] #[ignore] fn eventgraph_chaotic_propagation() { test_body!(eventgraph_chaotic_propagation_real); } async fn eventgraph_chaotic_propagation_real(ex: Arc>) { let mut rng = rand::thread_rng(); let peer_indexes: Vec = (0..N_NODES).collect(); let n_events: usize = 100000; // Bootstrap nodes let mut eg_instances = bootstrap_nodes(&peer_indexes, 14200, &mut rng, ex.clone()).await; // ========================================= // 1. Assert that everyone's DAG is the same // ========================================= assert_dags(&eg_instances, 1, &mut rng).await; // =========================================== // 2. Create multiple events on multiple nodes for i in 0..n_events { let random_node = eg_instances.choose(&mut rng).unwrap(); let event = Event::new(i.to_be_bytes().to_vec(), random_node).await; let current_genesis = random_node.current_genesis.read().await; let dag_name = current_genesis.id().to_string(); random_node.header_dag_insert(vec![event.header.clone()], &dag_name).await.unwrap(); random_node.dag_insert(slice::from_ref(&event), &dag_name).await.unwrap(); random_node.p2p.broadcast(&EventPut(event)).await; } info!("Waiting 5s for events propagation"); sleep(5).await; // ========================================== // 3. Assert that everyone has all the events // ========================================== assert_dags(&eg_instances, n_events + 1, &mut rng).await; // ============================================================ // 4. Start a new node and try to sync the DAG from other peers // ============================================================ { // Connect to N_CONNS random peers. let peer_indexes_to_connect: Vec<_> = peer_indexes.choose_multiple(&mut rng, N_CONNS).collect(); let mut peers = vec![]; for peer_index in peer_indexes_to_connect { let port = 14200 + peer_index; peers.push(Url::parse(&format!("tcp://127.0.0.1:{port}")).unwrap()); } let event_graph = spawn_node( vec![Url::parse(&format!("tcp://127.0.0.1:{}", 14200 + N_NODES + 1)).unwrap()], peers, ex.clone(), ) .await; eg_instances.push(event_graph.clone()); event_graph.p2p.clone().start().await.unwrap(); info!("Waiting 5s for new node connection"); sleep(5).await; event_graph.sync_selected(2, false).await.unwrap() } // ============================================================ // 5. Assert the new synced DAG has the same contents as others // ============================================================ assert_dags(&eg_instances, n_events + 1, &mut rng).await; // Stop the P2P network for eg in eg_instances.iter() { eg.p2p.clone().stop().await; } }