/* This file is part of DarkFi (https://dark.fi) * * Copyright (C) 2020-2024 Dyne.org foundation * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ //! Outbound connections session. Manages the creation of outbound sessions. //! Used to create an outbound session and to stop and start the session. //! //! Class consists of a weak pointer to the p2p interface and a vector of //! outbound connection slots. Using a weak pointer to p2p allows us to //! avoid circular dependencies. The vector of slots is wrapped in a mutex //! lock. This is switched on every time we instantiate a connection slot //! and insures that no other part of the program uses the slots at the //! same time. use std::{ sync::{ atomic::{AtomicU32, Ordering}, Arc, Weak, }, time::{Duration, Instant}, }; use async_trait::async_trait; use futures::stream::{FuturesUnordered, StreamExt}; use log::{debug, error, info, warn}; use smol::lock::Mutex; use url::Url; use super::{ super::{ channel::ChannelPtr, connector::Connector, dnet::{self, dnetev, DnetEvent}, hosts::{HostColor, HostState}, message::GetAddrsMessage, p2p::{P2p, P2pPtr}, }, Session, SessionBitFlag, SESSION_OUTBOUND, }; use crate::{ system::{sleep, timeout::timeout, CondVar, StoppableTask, StoppableTaskPtr}, Error, Result, }; pub type OutboundSessionPtr = Arc; /// Defines outbound connections session. pub struct OutboundSession { /// Weak pointer to parent p2p object pub(in crate::net) p2p: Weak, /// Outbound connection slots slots: Mutex>>, /// Peer discovery task peer_discovery: Arc, } impl OutboundSession { /// Create a new outbound session. pub(crate) fn new(p2p: Weak) -> OutboundSessionPtr { Arc::new_cyclic(|session| Self { p2p, slots: Mutex::new(Vec::new()), peer_discovery: PeerDiscovery::new(session.clone()), }) } /// Start the outbound session. Runs the channel connect loop. pub(crate) async fn start(self: Arc) { let n_slots = self.p2p().settings().read().await.outbound_connections; info!(target: "net::outbound_session", "[P2P] Starting {} outbound connection slots.", n_slots); // Activate mutex lock on connection slots. let mut slots = self.slots.lock().await; let mut futures = FuturesUnordered::new(); let self_ = Arc::downgrade(&self); for i in 0..n_slots as u32 { let slot = Slot::new(self_.clone(), i); futures.push(slot.clone().start()); slots.push(slot); } while (futures.next().await).is_some() {} self.peer_discovery.clone().start().await; } /// Stops the outbound session. pub(crate) async fn stop(&self) { debug!(target: "net::outbound_session", "Stopping outbound session.."); let slots = &*self.slots.lock().await; let mut futures = FuturesUnordered::new(); for slot in slots { futures.push(slot.clone().stop()); } while (futures.next().await).is_some() {} self.peer_discovery.clone().stop().await; debug!(target: "net::outbound_session", "Outbound session stopped!"); } pub async fn slot_info(&self) -> Vec { let mut info = Vec::new(); let slots = &*self.slots.lock().await; for slot in slots { info.push(slot.channel_id.load(Ordering::Relaxed)); } info } fn wakeup_peer_discovery(&self) { self.peer_discovery.notify() } async fn wakeup_slots(&self) { let slots = &*self.slots.lock().await; for slot in slots { slot.notify(); } } } #[async_trait] impl Session for OutboundSession { fn p2p(&self) -> P2pPtr { self.p2p.upgrade().unwrap() } fn type_id(&self) -> SessionBitFlag { SESSION_OUTBOUND } } struct Slot { slot: u32, process: StoppableTaskPtr, wakeup_self: CondVar, session: Weak, connector: Connector, // For debugging channel_id: AtomicU32, } impl Slot { fn new(session: Weak, slot: u32) -> Arc { let settings = session.upgrade().unwrap().p2p().settings(); Arc::new(Self { slot, process: StoppableTask::new(), wakeup_self: CondVar::new(), session: session.clone(), connector: Connector::new(settings, session), channel_id: AtomicU32::new(0), }) } async fn start(self: Arc) { let ex = self.p2p().executor(); self.process.clone().start( self.run(), |res| async { match res { Ok(()) | Err(Error::NetworkServiceStopped) => {} Err(e) => error!("net::outbound_session {}", e), } }, Error::NetworkServiceStopped, ex, ); } async fn stop(self: Arc) { self.connector.stop(); self.process.stop().await; } /// Address selection algorithm that works as follows: up to /// gold_count, select from the goldlist. Up to white_count, /// select from the whitelist. For all other slots, select from /// the greylist. If none of these preferences are satisfied, do /// peer discovery. /// /// Selecting from the greylist for some % of the slots is necessary /// and healthy since we require the network retains some unreliable /// connections. A network that purely favors uptime over unreliable /// connections may be vulnerable to sybil by attackers with good uptime. async fn fetch_addrs(&self) -> Option<(Url, u64)> { let hosts = self.p2p().hosts(); let slot = self.slot as usize; let container = &self.p2p().hosts().container; // Acquire Settings read lock let settings = self.p2p().settings().read_arc().await; let white_count = (settings.white_connect_percent * settings.outbound_connections) / 100; let gold_count = settings.gold_connect_count; let transports = settings.allowed_transports.clone(); let transport_mixing = settings.transport_mixing; let preference_strict = settings.slot_preference_strict; // Drop Settings read lock drop(settings); let grey_only = hosts.container.is_empty(HostColor::White) && hosts.container.is_empty(HostColor::Gold) && !hosts.container.is_empty(HostColor::Grey); // If we only have grey entries, select from the greylist. Otherwise, // use the preference defined in settings. let addrs = if grey_only && !preference_strict { container.fetch(HostColor::Grey, &transports, transport_mixing) } else if slot < gold_count { container.fetch(HostColor::Gold, &transports, transport_mixing) } else if slot < white_count { container.fetch(HostColor::White, &transports, transport_mixing) } else { container.fetch(HostColor::Grey, &transports, transport_mixing) }; hosts.check_addrs(addrs).await } // We first try to make connections to the addresses on our gold list. We then find some // whitelist connections according to the whitelist percent default. Finally, any remaining // connections we make from the greylist. async fn run(self: Arc) -> Result<()> { let hosts = self.p2p().hosts(); loop { // Activate the slot debug!( target: "net::outbound_session::try_connect()", "[P2P] Finding a host to connect to for outbound slot #{}", self.slot, ); // Do peer discovery if we don't have any peers on the Grey, White or Gold list // (first time connecting to the network). if hosts.container.is_empty(HostColor::Grey) && hosts.container.is_empty(HostColor::White) && hosts.container.is_empty(HostColor::Gold) { dnetev!(self, OutboundSlotSleeping, { slot: self.slot, }); self.wakeup_self.reset(); // Peer discovery self.session().wakeup_peer_discovery(); // Wait to be woken up by peer discovery self.wakeup_self.wait().await; continue } let addr = if let Some(addr) = self.fetch_addrs().await { debug!(target: "net::outbound_session::run()", "Fetched addr={}, slot #{}", addr.0, self.slot); addr } else { debug!(target: "net::outbound_session::run()", "No address found! Activating peer discovery..."); dnetev!(self, OutboundSlotSleeping, { slot: self.slot, }); self.wakeup_self.reset(); // Peer discovery self.session().wakeup_peer_discovery(); // Wait to be woken up by peer discovery self.wakeup_self.wait().await; continue }; let host = addr.0; let last_seen = addr.1; let slot = self.slot; info!( target: "net::outbound_session::try_connect()", "[P2P] Connecting outbound slot #{} [{}]", slot, host, ); dnetev!(self, OutboundSlotConnecting, { slot, addr: host.clone(), }); let (addr, channel) = match self.try_connect(host.clone(), last_seen).await { Ok(connect_info) => connect_info, Err(err) => { debug!( target: "net::outbound_session::try_connect()", "[P2P] Outbound slot #{} connection failed: {}", slot, err ); dnetev!(self, OutboundSlotDisconnected, { slot, err: err.to_string() }); self.channel_id.store(0, Ordering::Relaxed); continue } }; info!( target: "net::outbound_session::try_connect()", "[P2P] Outbound slot #{} connected [{}]", slot, addr ); dnetev!(self, OutboundSlotConnected, { slot: self.slot, addr: addr.clone(), channel_id: channel.info.id }); // At this point we've managed to connect. let stop_sub = channel.subscribe_stop().await?; // Setup new channel if let Err(err) = self.session().register_channel(channel.clone(), self.p2p().executor()).await { info!( target: "net::outbound_session", "[P2P] Outbound slot #{} disconnected: {}", slot, err ); dnetev!(self, OutboundSlotDisconnected, { slot: self.slot, err: err.to_string() }); self.channel_id.store(0, Ordering::Relaxed); warn!( target: "net::outbound_session::try_connect()", "[P2P] Suspending addr=[{}] slot #{}", addr, slot ); // At this point we failed to connect. We'll downgrade this peer now. self.p2p().hosts().move_host(&addr, last_seen, HostColor::Grey)?; // Mark its state as Suspend, which sends this node to the Refinery for processing. self.p2p().hosts().try_register(addr.clone(), HostState::Suspend).unwrap(); continue } self.channel_id.store(channel.info.id, Ordering::Relaxed); // Wait for channel to close stop_sub.receive().await; self.channel_id.store(0, Ordering::Relaxed); } } /// Start making an outbound connection, using provided [`Connector`]. /// Tries to find a valid address to connect to, otherwise does peer /// discovery. The peer discovery loops until some peer we can connect /// to is found. Once connected, registers the channel, removes it from /// the list of pending channels, and starts sending messages across the /// channel. In case of any failures, a network error is returned and the /// main connect loop (parent of this function) will iterate again. async fn try_connect(&self, addr: Url, last_seen: u64) -> Result<(Url, ChannelPtr)> { match self.connector.connect(&addr).await { Ok((addr_final, channel)) => Ok((addr_final, channel)), Err(err) => { info!( target: "net::outbound_session::try_connect()", "[P2P] Unable to connect outbound slot #{} [{}]: {}", self.slot, addr, err ); // Immediately return if the Connector has stopped. // This indicates a shutdown of the P2P network and // should not result in hostlist modifications. if let Error::ConnectorStopped = err { return Err(Error::ConnectFailed); } // At this point we failed to connect. We'll downgrade this peer now. self.p2p().hosts().move_host(&addr, last_seen, HostColor::Grey)?; // Mark its state as Suspend, which sends it to the Refinery for processing. self.p2p().hosts().try_register(addr.clone(), HostState::Suspend).unwrap(); // Notify that channel processing failed self.p2p().hosts().channel_publisher.notify(Err(Error::ConnectFailed)).await; Err(Error::ConnectFailed) } } } fn notify(&self) { self.wakeup_self.notify() } fn session(&self) -> OutboundSessionPtr { self.session.upgrade().unwrap() } fn p2p(&self) -> P2pPtr { self.session().p2p() } } /// Defines a common interface for multiple peer discovery processes. /// /// NOTE: Currently only one Peer Discovery implementation exists. Making /// Peer Discovery generic enables us to support network swarming, since /// the peer discovery process will differ depending on whether it occurs /// on the overlay network or a subnet. #[async_trait] pub trait PeerDiscoveryBase { async fn start(self: Arc); async fn stop(self: Arc); async fn run(self: Arc); async fn wait(&self) -> bool; fn notify(&self); fn session(&self) -> OutboundSessionPtr; fn p2p(&self) -> P2pPtr; } /// Main PeerDiscovery process that loops through connected peers /// and sends out a `GetAddrs` when it is active. If there are no /// connected peers after two attempts, connect to our seed nodes /// and perform `SeedSyncSession`. struct PeerDiscovery { process: StoppableTaskPtr, wakeup_self: CondVar, session: Weak, } impl PeerDiscovery { fn new(session: Weak) -> Arc { Arc::new(Self { process: StoppableTask::new(), wakeup_self: CondVar::new(), session }) } } #[async_trait] impl PeerDiscoveryBase for PeerDiscovery { async fn start(self: Arc) { let ex = self.p2p().executor(); self.process.clone().start( async move { self.run().await; unreachable!(); }, // Ignore stop handler |_| async {}, Error::NetworkServiceStopped, ex, ); } async fn stop(self: Arc) { self.process.stop().await; } /// Activate peer discovery if not active already. For the first two /// attempts, this will loop through all connected P2P peers and send /// out a `GetAddrs` message to request more peers. Other parts of the /// P2P stack will then handle the incoming addresses and place them in /// the hosts list. /// /// On the third attempt, and if we still haven't made any connections, /// this function will then call `p2p.seed()` which triggers a /// `SeedSyncSession` that will connect to configured seeds and request /// peers from them. /// /// This function will also sleep `outbound_peer_discovery_attempt_time` /// seconds after broadcasting in order to let the P2P stack receive and /// work through the addresses it is expecting. async fn run(self: Arc) { let mut current_attempt = 0; loop { dnetev!(self, OutboundPeerDiscovery, { attempt: current_attempt, state: "wait", }); // wait to be woken up by notify() let sleep_was_instant = self.wait().await; // Read the current P2P settings let settings = self.p2p().settings().read_arc().await; let outbound_peer_discovery_cooloff_time = settings.outbound_peer_discovery_cooloff_time; let outbound_peer_discovery_attempt_time = settings.outbound_peer_discovery_attempt_time; let outbound_connections = settings.outbound_connections; let allowed_transports = settings.allowed_transports.clone(); let seeds = settings.seeds.clone(); drop(settings); if sleep_was_instant { // Try again current_attempt += 1; } else { // reset back to start current_attempt = 1; } if current_attempt >= 4 { info!( target: "net::outbound_session::peer_discovery()", "[P2P] [PEER DISCOVERY] Sleeping and trying again. Attempt {current_attempt}" ); dnetev!(self, OutboundPeerDiscovery, { attempt: current_attempt, state: "sleep", }); sleep(outbound_peer_discovery_cooloff_time).await; current_attempt = 1; } // First 2 times try sending GetAddr to the network. // 3rd time do a seed sync (providing we have seeds // configured). if self.p2p().is_connected() && current_attempt <= 2 { // Broadcast the GetAddrs message to all active peers. // If we have no active peers, we will perform a SeedSyncSession instead. info!( target: "net::outbound_session::peer_discovery()", "[P2P] [PEER DISCOVERY] Asking peers for new peers to connect to..."); dnetev!(self, OutboundPeerDiscovery, { attempt: current_attempt, state: "getaddr", }); let get_addrs = GetAddrsMessage { max: outbound_connections as u32, transports: allowed_transports, }; self.p2p().broadcast(&get_addrs).await; // Wait for a hosts store update event let store_sub = self.p2p().hosts().subscribe_store().await; let result = timeout( Duration::from_secs(outbound_peer_discovery_attempt_time), store_sub.receive(), ) .await; match result { Ok(addrs_len) => { info!( target: "net::outbound_session::peer_discovery()", "[P2P] [PEER DISCOVERY] Discovered {} peers", addrs_len ); } Err(_) => { warn!( target: "net::outbound_session::peer_discovery()", "[P2P] [PEER DISCOVERY] Waiting for addrs timed out." ); // Just do seed next time current_attempt = 3; } } // NOTE: not every call to subscribe() in net/ has a // corresponding unsubscribe(). To do this we need async // Drop. For now it's sufficient for publishers to be // de-allocated when the Session completes. store_sub.unsubscribe().await; } else if !seeds.is_empty() { info!( target: "net::outbound_session::peer_discovery()", "[P2P] [PEER DISCOVERY] Asking seeds for new peers to connect to..."); dnetev!(self, OutboundPeerDiscovery, { attempt: current_attempt, state: "seed", }); self.p2p().seed().await; } self.wakeup_self.reset(); self.session().wakeup_slots().await; // Give some time for new connections to be established sleep(outbound_peer_discovery_attempt_time).await; } } /// Blocks execution until we receive a notification from notify(). /// `wakeup_self.wait()` resets the condition variable (`CondVar`) and waits /// for a call from `notify()`. Returns `true` if the function completed /// instantly (i.e. no wait occured). Returns false otherwise. async fn wait(&self) -> bool { let wakeup_start = Instant::now(); self.wakeup_self.wait().await; let wakeup_end = Instant::now(); let epsilon = Duration::from_millis(200); wakeup_end - wakeup_start <= epsilon } /// Wakeup peer discovery by sending a notification to `wakeup_self`. /// Uses the underlying `CondVar` method `notify()`. Subsequent calls /// to this do nothing until `wait()` is called. fn notify(&self) { self.wakeup_self.notify() } fn session(&self) -> OutboundSessionPtr { self.session.upgrade().unwrap() } fn p2p(&self) -> P2pPtr { self.session().p2p() } }