lilith+net: move whitelist_refinery back into lilith + create new public functions

we expose some methods that allow uses of the P2P interface, such as
lilith, to interact with the hostlist in a safe way.

bin/lilith/src/main.rs

@@ -20,10 +20,11 @@ use std::{
     collections::{HashMap, HashSet},
     process::exit,
     sync::Arc,
+    time::UNIX_EPOCH,
 };
 
 use async_trait::async_trait;
-use log::{error, info, warn};
+use log::{debug, error, info, warn};
 use semver::Version;
 use smol::{
     lock::{Mutex, MutexGuard},
@@ -38,12 +39,12 @@ use url::Url;
 
 use darkfi::{
     async_daemonize, cli_desc,
-    net::{self, hosts::HostColor, session::whitelist_refinery, P2p, P2pPtr},
+    net::{self, hosts::HostColor, P2p, P2pPtr},
     rpc::{
         jsonrpc::*,
         server::{listen_and_serve, RequestHandler},
     },
-    system::{StoppableTask, StoppableTaskPtr},
+    system::{sleep, StoppableTask, StoppableTaskPtr},
     util::path::get_config_path,
     Error, Result,
 };
@@ -70,6 +71,10 @@ struct Args {
     #[structopt(short, parse(from_occurrences))]
     /// Increase verbosity (-vvv supported)
     pub verbose: u8,
+
+    #[structopt(long, default_value = "120")]
+    /// Interval after which to check whitelist peers
+    whitelist_refinery_interval: u64,
 }
 
 /// Struct representing a spawned P2P network
@@ -156,6 +161,70 @@ struct Lilith {
 }
 
 impl Lilith {
+    /// Since `Lilith` does not make outbound connections, if a peer is
+    /// upgraded to whitelist it will remain on the whitelist even if the
+    /// give peer is no longer online.
+    ///
+    /// To protect `Lilith` from sharing potentially offline nodes,
+    /// `whitelist_refinery` periodically ping nodes on the whitelist. If they
+    /// are reachable, we update their last seen field. Otherwise, we downgrade
+    /// them to the greylist.
+    ///
+    /// Note: if `Lilith` loses connectivity this method will delete peers from
+    /// the whitelist, meaning `Lilith` will need to rebuild its hostlist when
+    /// it comes back online.
+    async fn whitelist_refinery(
+        network_name: String,
+        p2p: P2pPtr,
+        refinery_interval: u64,
+    ) -> Result<()> {
+        debug!(target: "net::refinery::whitelist_refinery", "Starting whitelist refinery for \"{}\"",
+           network_name);
+
+        let hosts = p2p.hosts();
+
+        loop {
+            sleep(refinery_interval).await;
+
+            if hosts.container.is_empty(HostColor::White).await {
+                warn!(target: "net::refinery::whitelist_refinery",
+                      "Whitelist is empty! Cannot start refinery process");
+
+                continue
+            }
+
+            let (entry, position) = hosts.container.fetch_last(HostColor::White).await;
+
+            let url = &entry.0;
+            let last_seen = &entry.1;
+
+            if !hosts.refinable(url.clone()).await {
+                debug!(target: "net::refinery::whitelist_refinery", "Addr={} not available!",
+                       url.clone());
+
+                continue
+            }
+
+            if p2p.session_refine().handshake_node(url.clone(), p2p.clone()).await {
+                debug!(target: "net::refinery:::whitelist_refinery",
+                       "Host {} is not responsive. Downgrading from whitelist", url);
+
+                hosts.greylist_host(url, *last_seen).await?;
+
+                continue
+            }
+
+            debug!(target: "net::refinery::whitelist_refinery",
+                   "Peer {} is responsive. Updating last_seen", url);
+
+            // This node is active. Update the last seen field.
+            let last_seen = UNIX_EPOCH.elapsed().unwrap().as_secs();
+            hosts
+                .container
+                .update_last_seen(HostColor::White as usize, url, last_seen, Some(position))
+                .await;
+        }
+    }
     // RPCAPI:
     // Returns all spawned networks names with their node addresses.
     // --> {"jsonrpc": "2.0", "method": "spawns", "params": [], "id": 42}
@@ -337,7 +406,7 @@ async fn realmain(args: Args, ex: Arc<Executor<'static>>) -> Result<()> {
         let name = network.name.clone();
         let task = StoppableTask::new();
         task.clone().start(
-            whitelist_refinery(name.clone(), network.p2p.clone()),
+            Lilith::whitelist_refinery(name.clone(), network.p2p.clone(), args.whitelist_refinery_interval),
             |res| async move {
                 match res {
                     Ok(()) | Err(Error::DetachedTaskStopped) => { /* Do nothing */ }

src/net/hosts.rs

@@ -386,7 +386,7 @@ impl HostContainer {
     }
 
     /// Update the last_seen field of a peer on a hostlist.
-    pub(in crate::net) async fn update_last_seen(
+    pub async fn update_last_seen(
         &self,
         color: usize,
         addr: &Url,
@@ -850,6 +850,12 @@ impl Hosts {
         trace!(target: "net::hosts:insert()", "[END]");
     }
 
+    /// Check whether a peer is available to be refined currently. Returns true
+    /// if available, false otherwise.
+    pub async fn refinable(&self, addr: Url) -> bool {
+        self.try_register(addr.clone(), HostState::Refine).await.is_ok()
+    }
+
     /// Try to update the registry. If the host already exists, try to update its state.
     /// Otherwise add the host to the registry along with its state.
     pub(in crate::net) async fn try_register(
@@ -1192,6 +1198,17 @@ impl Hosts {
         }
     }
 
+    /// Downgrade host to Greylist, remove from Gold or White list.
+    pub async fn greylist_host(&self, addr: &Url, last_seen: u64) -> Result<()> {
+        debug!(target: "net::hosts:greylist_host()", "Downgrading addr={}", addr);
+        self.move_host(addr, last_seen, HostColor::Grey).await?;
+
+        // Free up this addr for future operations.
+        self.unregister(addr).await;
+
+        Ok(())
+    }
+
     /// A single atomic function for moving hosts between hostlists. Called on the following occasions:
     ///
     /// * When we cannot connect to a peer: move to grey, remove from white and gold.

src/net/session/mod.rs

@@ -37,7 +37,7 @@ pub use outbound_session::{OutboundSession, OutboundSessionPtr};
 pub mod seedsync_session;
 pub use seedsync_session::{SeedSyncSession, SeedSyncSessionPtr};
 pub mod refine_session;
-pub use refine_session::{whitelist_refinery, RefineSession, RefineSessionPtr};
+pub use refine_session::{RefineSession, RefineSessionPtr};
 
 /// Bitwise selectors for the `protocol_registry`
 pub type SessionBitFlag = u32;

src/net/session/refine_session.rs

@@ -48,11 +48,9 @@ use crate::{
         session::{Session, SessionBitFlag, SESSION_REFINE},
     },
     system::{sleep, timeout::timeout, LazyWeak, StoppableTask, StoppableTaskPtr},
-    Error, Result,
+    Error,
 };
 
-// TODO: Make this configurable
-const WHITELIST_REFINERY_INTERVAL: u64 = 60;
 pub type RefineSessionPtr = Arc<RefineSession>;
 
 pub struct RefineSession {
@@ -125,8 +123,7 @@ impl RefineSession {
 
     /// Globally accessible function to perform a version exchange with a
     /// given address.  Returns `true` if an address is accessible, false
-    /// otherwise.  Used by `GreylistRefinery`, `SelfHandshake`, and in
-    /// `Lilith`, which contains an implemenenation of a whitelist refinery.
+    /// otherwise.  
     pub async fn handshake_node(self: Arc<Self>, addr: Url, p2p: P2pPtr) -> bool {
         let self_ = Arc::downgrade(&self);
         let connector = Connector::new(self.p2p().settings(), self_);
@@ -333,58 +330,6 @@ impl GreylistRefinery {
     }
 }
 
-/// Periodically ping nodes on the whitelist. If they are still reachable, update their last
-/// seen field. Otherwise, downgrade them to the greylist.
-pub async fn whitelist_refinery(network_name: String, p2p: P2pPtr) -> Result<()> {
-    debug!(target: "net::refinery::whitelist_refinery", "Starting whitelist refinery for \"{}\"",
-           network_name);
-
-    let hosts = p2p.hosts();
-
-    loop {
-        sleep(WHITELIST_REFINERY_INTERVAL).await;
-
-        if hosts.container.is_empty(HostColor::White).await {
-            warn!(target: "net::refinery::whitelist_refinery",
-                      "Whitelist is empty! Cannot start refinery process");
-
-            continue
-        }
-
-        let (entry, position) = hosts.container.fetch_last(HostColor::White).await;
-
-        let url = &entry.0;
-        let last_seen = &entry.1;
-
-        if let Err(e) = hosts.try_register(url.clone(), HostState::Refine).await {
-            debug!(target: "net::refinery::whitelist_refinery", "Unable to refine addr={}, err={}",
-                           url.clone(), e);
-
-            continue
-        }
-
-        if p2p.session_refine().handshake_node(url.clone(), p2p.clone()).await {
-            debug!(target: "net::refinery:::whitelist_refinery",
-                       "Host {} is not responsive. Downgrading from whitelist", url);
-
-            hosts.move_host(url, *last_seen, HostColor::Grey).await?;
-            hosts.unregister(url).await;
-
-            continue
-        }
-
-        debug!(target: "net::refinery::whitelist_refinery",
-                   "Peer {} is responsive. Updating last_seen", url);
-
-        // This node is active. Update the last seen field.
-        let last_seen = UNIX_EPOCH.elapsed().unwrap().as_secs();
-        hosts
-            .container
-            .update_last_seen(HostColor::White as usize, url, last_seen, Some(position))
-            .await;
-    }
-}
-
 /// Periodically try to do a version exchange with our own external
 /// addresses. If the version exchange is successful, take a timestamp and
 /// save it along with the external addresses. Each address along with its