net: simplify and reduce Hosts API by introducing HostContainer

HostContainer is an Array of 4 hostlists which are identified by
HostColor as Grey, White, Gold or Black.

Using this Container allows us to quickly replicate the same utilities
on all host functions while minimizing code reuse.

bin/lilith/src/main.rs

@@ -41,7 +41,10 @@ use darkfi::{
     async_daemonize, cli_desc,
     net::{
         self,
-        hosts::{refinery::ping_node, store::HostState},
+        hosts::{
+            refinery::ping_node,
+            store::{HostColor, HostState},
+        },
         P2p, P2pPtr,
     },
     rpc::{
@@ -92,7 +95,8 @@ impl Spawn {
     async fn get_whitelist(&self) -> Vec<JsonValue> {
         self.p2p
             .hosts()
-            .whitelist_fetch_all()
+            .container
+            .fetch_all(HostColor::White)
             .await
             .iter()
             .map(|(addr, _url)| JsonValue::String(addr.to_string()))
@@ -102,7 +106,8 @@ impl Spawn {
     async fn get_greylist(&self) -> Vec<JsonValue> {
         self.p2p
             .hosts()
-            .greylist_fetch_all()
+            .container
+            .fetch_all(HostColor::Grey)
             .await
             .iter()
             .map(|(addr, _url)| JsonValue::String(addr.to_string()))
@@ -112,12 +117,14 @@ impl Spawn {
     async fn get_anchorlist(&self) -> Vec<JsonValue> {
         self.p2p
             .hosts()
-            .anchorlist_fetch_all()
+            .container
+            .fetch_all(HostColor::Gold)
             .await
             .iter()
             .map(|(addr, _url)| JsonValue::String(addr.to_string()))
             .collect()
     }
+
     async fn info(&self) -> JsonValue {
         let mut addr_vec = vec![];
         for addr in &self.p2p.settings().inbound_addrs {
@@ -171,24 +178,28 @@ impl Lilith {
         loop {
             sleep(REFINERY_INTERVAL).await;
 
-            if hosts.is_empty_whitelist().await {
+            if hosts.container.is_empty(HostColor::White).await {
                 warn!(target: "lilith", "Whitelist is empty! Cannot start refinery process");
 
                 continue
             }
 
-            let (entry, position) = hosts.whitelist_fetch_last().await;
+            let (entry, position) = hosts.container.fetch_last(HostColor::White).await;
             let url = &entry.0;
 
-            if let Err(_) = hosts.try_update_registry(url.clone(), HostState::Refining).await {
+            if let Err(_) = hosts.try_register(url.clone(), HostState::Refining).await {
                 continue
             }
 
             if !ping_node(url.clone(), p2p.clone()).await {
-                let (_addr, last_seen) = hosts.get_whitelist_entry_at_addr(url).await.unwrap();
-                hosts.greylist_store_or_update(&[(url.clone(), last_seen)]).await;
-
-                hosts.whitelist_remove(url, position).await;
+                let (_addr, last_seen) = hosts
+                    .container
+                    .get_entry_at_addr(HostColor::White as usize, url)
+                    .await
+                    .unwrap();
+                hosts.insert(HostColor::Grey, &[(url.clone(), last_seen)]).await;
+
+                hosts.container.remove(HostColor::White, url, position).await;
                 debug!(target: "lilith", "Host {} is not responsive. Downgraded from whitelist", url);
 
                 // Remove this entry from HostRegistry to avoid this host getting
@@ -196,14 +207,17 @@ impl Lilith {
                 //
                 // It is not necessary to call this when the refinery passes, since the
                 // state will be changed to Connected.
-                hosts.remove(url).await;
+                hosts.unregister(url).await;
 
                 continue
             }
 
             // This node is active. Update the last seen field.
             let last_seen = UNIX_EPOCH.elapsed().unwrap().as_secs();
-            hosts.whitelist_update_last_seen(url, last_seen, position).await;
+            hosts
+                .container
+                .update_last_seen(HostColor::White as usize, url, last_seen, position)
+                .await;
         }
     }
 

src/net/acceptor.rs

@@ -30,6 +30,7 @@ use url::Url;
 
 use super::{
     channel::{Channel, ChannelPtr},
+    hosts::store::HostColor,
     session::SessionWeakPtr,
     transport::{Listener, PtListener},
 };
@@ -117,7 +118,16 @@ impl Acceptor {
             match listener.next().await {
                 Ok((stream, url)) => {
                     // Check if we reject this peer
-                    if self.session.upgrade().unwrap().p2p().hosts().is_blacklist(&url).await {
+                    if self
+                        .session
+                        .upgrade()
+                        .unwrap()
+                        .p2p()
+                        .hosts()
+                        .container
+                        .contains(HostColor::Black as usize, &url)
+                        .await
+                    {
                         warn!(target: "net::acceptor::run_accept_loop()", "Peer {} is blacklisted", url);
                         continue
                     }

src/net/channel.rs

@@ -311,7 +311,8 @@ impl Channel {
     /// Ban a malicious peer and stop the channel.
     pub async fn ban(&self, peer: &Url) {
         debug!(target: "net::channel::ban()", "START {:?}", self);
-        self.p2p().hosts().blacklist(peer).await;
+        self.p2p().hosts().blacklist(&peer).await;
+
         self.stop().await;
         debug!(target: "net::channel::ban()", "STOP {:?}", self);
     }

src/net/connector.rs

@@ -23,6 +23,7 @@ use url::Url;
 
 use super::{
     channel::{Channel, ChannelPtr},
+    hosts::store::HostColor,
     session::SessionWeakPtr,
     settings::SettingsPtr,
     transport::Dialer,
@@ -45,7 +46,16 @@ impl Connector {
 
     /// Establish an outbound connection
     pub async fn connect(&self, url: &Url) -> Result<(Url, ChannelPtr)> {
-        if self.session.upgrade().unwrap().p2p().hosts().is_blacklist(url).await {
+        if self
+            .session
+            .upgrade()
+            .unwrap()
+            .p2p()
+            .hosts()
+            .container
+            .contains(HostColor::Black as usize, url)
+            .await
+        {
             warn!(target: "net::connector::connect", "Peer {} is blacklisted", url);
             return Err(Error::ConnectFailed)
         }

src/net/hosts/mod.rs

@@ -32,6 +32,7 @@
 /// and `ProtocolAddress`.
 pub mod refinery;
 
+/// TODO: update documentation
 /// The main interface for interacting with the hostlist.
 ///
 /// The hostlist is stored in three sections: white, grey, and anchorlists.

src/net/hosts/refinery.rs

@@ -24,7 +24,10 @@ use std::{
 use log::{debug, warn};
 use url::Url;
 
-use super::super::p2p::{P2p, P2pPtr};
+use super::{
+    super::p2p::{P2p, P2pPtr},
+    store::HostColor,
+};
 use crate::{
     net::{
         connector::Connector, hosts::store::HostState, protocol::ProtocolVersion, session::Session,
@@ -52,7 +55,7 @@ impl GreylistRefinery {
     }
 
     pub async fn start(self: Arc<Self>) {
-        match self.p2p().hosts().load_hosts().await {
+        match self.p2p().hosts().container.load_all(&self.p2p().settings().hostlist).await {
             Ok(()) => {
                 debug!(target: "net::refinery::start()", "Load hosts successful!");
             }
@@ -76,7 +79,7 @@ impl GreylistRefinery {
     pub async fn stop(self: Arc<Self>) {
         self.process.stop().await;
 
-        match self.p2p().hosts().save_hosts().await {
+        match self.p2p().hosts().container.save_all(&self.p2p().settings().hostlist).await {
             Ok(()) => {
                 debug!(target: "net::refinery::stop()", "Save hosts successful!");
             }
@@ -95,7 +98,7 @@ impl GreylistRefinery {
 
             let hosts = self.p2p().hosts();
 
-            if hosts.is_empty_greylist().await {
+            if hosts.container.is_empty(HostColor::Grey).await {
                 debug!(target: "net::refinery",
                 "Greylist is empty! Cannot start refinery process");
 
@@ -103,17 +106,22 @@ impl GreylistRefinery {
             }
 
             // Only attempt to refine peers that match our transports.
-            match hosts.greylist_fetch_random_with_schemes().await {
+            match hosts
+                .container
+                .fetch_random_with_schemes(
+                    HostColor::Grey,
+                    &self.p2p().settings().allowed_transports,
+                )
+                .await
+            {
                 Some((entry, position)) => {
                     let url = &entry.0;
 
-                    if let Err(_) =
-                        hosts.try_update_registry(url.clone(), HostState::Refining).await
-                    {
+                    if let Err(_) = hosts.try_register(url.clone(), HostState::Refining).await {
                         continue
                     }
                     if !ping_node(url.clone(), self.p2p().clone()).await {
-                        hosts.greylist_remove(url, position).await;
+                        hosts.container.remove(HostColor::Grey, url, position).await;
 
                         debug!(
                             target: "net::refinery",
@@ -125,7 +133,7 @@ impl GreylistRefinery {
                         //
                         // It is not necessary to call this when the refinery passes, since the
                         // state will be changed to Connected.
-                        hosts.remove(url).await;
+                        hosts.unregister(url).await;
 
                         continue
                     }
@@ -133,10 +141,13 @@ impl GreylistRefinery {
                     let last_seen = UNIX_EPOCH.elapsed().unwrap().as_secs();
 
                     // Append to the whitelist.
-                    hosts.whitelist_store_or_update(&[(url.clone(), last_seen)]).await;
+                    hosts
+                        .container
+                        .store_or_update(HostColor::White, &[(url.clone(), last_seen)])
+                        .await;
 
                     // Remove whitelisted peer from the greylist.
-                    hosts.greylist_remove(url, position).await;
+                    hosts.container.remove(HostColor::Grey, url, position).await;
                 }
                 None => {
                     debug!(target: "net::refinery", "No matching greylist entries found. Cannot proceed with refinery");

src/net/protocol/protocol_address.rs

@@ -25,7 +25,7 @@ use smol::Executor;
 use super::{
     super::{
         channel::ChannelPtr,
-        hosts::store::HostsPtr,
+        hosts::store::{HostColor, HostsPtr},
         message::{AddrsMessage, GetAddrsMessage},
         message_subscriber::MessageSubscription,
         p2p::P2pPtr,
@@ -118,7 +118,7 @@ impl ProtocolAddress {
                 "Appending to greylist...",
             );
 
-            self.hosts.greylist_store_or_update(&addrs_msg.addrs).await;
+            self.hosts.insert(HostColor::Grey, &addrs_msg.addrs).await;
         }
     }
 
@@ -157,7 +157,9 @@ impl ProtocolAddress {
             "Fetching anchorlist entries with schemes");
             let mut addrs = self
                 .hosts
-                .anchorlist_fetch_n_random_with_schemes(
+                .container
+                .fetch_n_random_with_schemes(
+                    HostColor::Gold,
                     &get_addrs_msg.transports,
                     get_addrs_msg.max,
                 )
@@ -169,7 +171,9 @@ impl ProtocolAddress {
             addrs.append(
                 &mut self
                     .hosts
-                    .whitelist_fetch_n_random_with_schemes(
+                    .container
+                    .fetch_n_random_with_schemes(
+                        HostColor::White,
                         &get_addrs_msg.transports,
                         get_addrs_msg.max,
                     )
@@ -184,7 +188,12 @@ impl ProtocolAddress {
             addrs.append(
                 &mut self
                     .hosts
-                    .whitelist_fetch_n_random_excluding_schemes(&get_addrs_msg.transports, remain)
+                    .container
+                    .fetch_n_random_excluding_schemes(
+                        HostColor::White,
+                        &get_addrs_msg.transports,
+                        remain,
+                    )
                     .await,
             );
 
@@ -193,7 +202,7 @@ impl ProtocolAddress {
             debug!(target: "net::protocol_address::handle_receive_get_addrs()",
             "Fetching greylist entries");
             let remain = 2 * get_addrs_msg.max - addrs.len() as u32;
-            addrs.append(&mut self.hosts.greylist_fetch_n_random(remain).await);
+            addrs.append(&mut self.hosts.container.fetch_n_random(HostColor::Grey, remain).await);
 
             debug!(
                 target: "net::protocol_address::handle_receive_get_addrs()",

src/net/protocol/protocol_seed.rs

@@ -25,7 +25,7 @@ use smol::Executor;
 use super::{
     super::{
         channel::ChannelPtr,
-        hosts::store::HostsPtr,
+        hosts::store::{HostColor, HostsPtr},
         message::{AddrsMessage, GetAddrsMessage},
         message_subscriber::MessageSubscription,
         p2p::P2pPtr,
@@ -123,7 +123,7 @@ impl ProtocolBase for ProtocolSeed {
             target: "net::protocol_seed::start()",
             "Appending to greylist...",
         );
-        self.hosts.greylist_store_or_update(&addrs_msg.addrs).await;
+        self.hosts.insert(HostColor::Grey, &addrs_msg.addrs).await;
 
         debug!(target: "net::protocol_seed::start()", "END => address={}", self.channel.address());
         Ok(())

src/net/session/inbound_session.rs

@@ -199,7 +199,7 @@ impl InboundSession {
 
         stop_sub.receive().await;
 
-        self.p2p().hosts().remove(channel.clone().address()).await;
+        self.p2p().hosts().unregister(channel.clone().address()).await;
 
         debug!(
             target: "net::inbound_session::setup_channel()",

src/net/session/manual_session.rs

@@ -32,7 +32,7 @@
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use log::{debug, info, warn};
+use log::{info, warn};
 use smol::lock::Mutex;
 use url::Url;
 
@@ -114,8 +114,7 @@ impl ManualSession {
                 addr, tried_attempts,
             );
 
-            if let Err(e) =
-                self.p2p().hosts().try_update_registry(addr.clone(), HostState::Pending).await
+            if let Err(e) = self.p2p().hosts().try_register(addr.clone(), HostState::Pending).await
             {
                 warn!(target: "net::manual_session", "{}", e);
             }
@@ -136,7 +135,7 @@ impl ManualSession {
                     self.register_channel(channel.clone(), ex.clone()).await?;
 
                     // Add this connection to the anchorlist
-                    self.p2p().hosts().upgrade_host(&addr).await;
+                    self.p2p().hosts().container.upgrade_host(&addr).await;
 
                     // Notify that channel processing has finished
                     self.channel_subscriber.notify(Ok(channel)).await;
@@ -185,7 +184,7 @@ impl ManualSession {
         );
         // Stop tracking this address in the HostRegistry.
         // Otherwise, host will be stuck in Pending state.
-        self.p2p().hosts().remove(&addr).await;
+        self.p2p().hosts().unregister(&addr).await;
 
         Ok(())
     }

src/net/session/mod.rs

@@ -62,7 +62,7 @@ pub async fn remove_sub_on_stop(p2p: P2pPtr, channel: ChannelPtr) {
     );
 
     // Remove channel from p2p
-    p2p.hosts().remove(channel.address()).await;
+    p2p.hosts().unregister(channel.address()).await;
     debug!(target: "net::session::remove_sub_on_stop()", "[END]");
 }
 
@@ -145,9 +145,9 @@ pub trait Session: Sync {
         protocol_version.run(executor.clone()).await?;
 
         // Attempt to add channel to registry
-        if let Err(e) = self.p2p().hosts().store(channel.clone()).await {
+        if let Err(e) = self.p2p().hosts().register_channel(channel.clone()).await {
             warn!(target: "net::session::perform_handshake_protocols()",
-            "Couldn't add channel {} to registry!! {}", channel.address(), e);
+            "Couldn't add channel {} to registry! {}", channel.address(), e);
             return Err(e)
         }
 

src/net/session/outbound_session.rs

@@ -44,6 +44,7 @@ use super::{
         channel::ChannelPtr,
         connector::Connector,
         dnet::{self, dnetev, DnetEvent},
+        hosts::store::HostColor,
         message::GetAddrsMessage,
         p2p::{P2p, P2pPtr},
     },
@@ -190,6 +191,7 @@ impl Slot {
         let hosts = self.p2p().hosts();
         let connects = self.p2p().settings().outbound_connections;
         let white_count = connects * self.p2p().settings().white_connection_percent / 100;
+        let transport_mixing = self.p2p().settings().transport_mixing;
 
         if slot_count < self.p2p().settings().anchor_connection_count {
             //  Up to anchor_connection_count connections:
@@ -197,20 +199,44 @@ impl Slot {
             //  If the anchorlist is empty, select from the whitelist
             //  If the whitelist is empty, select from the greylist
             //  If the greylist is empty, return None and do peer discovery
-            if !hosts.anchorlist_fetch_address(transports).await.is_empty() {
-                let addrs = hosts.anchorlist_fetch_address(transports).await;
+            if !hosts
+                .container
+                .fetch_address(HostColor::Gold, transports, transport_mixing)
+                .await
+                .is_empty()
+            {
+                let addrs = hosts
+                    .container
+                    .fetch_address(HostColor::Gold, transports, transport_mixing)
+                    .await;
 
                 return hosts.check_address(addrs).await
             }
 
-            if !hosts.whitelist_fetch_address(transports).await.is_empty() {
-                let addrs = hosts.whitelist_fetch_address(transports).await;
+            if !hosts
+                .container
+                .fetch_address(HostColor::White, transports, transport_mixing)
+                .await
+                .is_empty()
+            {
+                let addrs = hosts
+                    .container
+                    .fetch_address(HostColor::White, transports, transport_mixing)
+                    .await;
 
                 return hosts.check_address(addrs).await
             }
 
-            if !hosts.greylist_fetch_address(transports).await.is_empty() {
-                let addrs = hosts.greylist_fetch_address(transports).await;
+            if !hosts
+                .container
+                .fetch_address(HostColor::Grey, transports, transport_mixing)
+                .await
+                .is_empty()
+            {
+                let addrs = hosts
+                    .container
+                    .fetch_address(HostColor::Grey, transports, transport_mixing)
+                    .await;
 
                 return hosts.check_address(addrs).await
             }
@@ -219,14 +245,30 @@ impl Slot {
             //  Select from the whitelist
             //  If the whitelist is empty, select from the greylist
             //  If the greylist is empty, return None and do peer discovery
-            if !hosts.whitelist_fetch_address(transports).await.is_empty() {
-                let addrs = hosts.whitelist_fetch_address(transports).await;
+            if !hosts
+                .container
+                .fetch_address(HostColor::White, transports, transport_mixing)
+                .await
+                .is_empty()
+            {
+                let addrs = hosts
+                    .container
+                    .fetch_address(HostColor::White, transports, transport_mixing)
+                    .await;
 
                 return hosts.check_address(addrs).await
             }
 
-            if !hosts.greylist_fetch_address(transports).await.is_empty() {
-                let addrs = hosts.greylist_fetch_address(transports).await;
+            if !hosts
+                .container
+                .fetch_address(HostColor::Grey, transports, transport_mixing)
+                .await
+                .is_empty()
+            {
+                let addrs = hosts
+                    .container
+                    .fetch_address(HostColor::Grey, transports, transport_mixing)
+                    .await;
 
                 return hosts.check_address(addrs).await
             }
@@ -234,8 +276,16 @@ impl Slot {
             // All other connections:
             //  Select from the greylist
             //  If the greylist is empty, do peer discovery
-            if !hosts.greylist_fetch_address(transports).await.is_empty() {
-                let addrs = hosts.greylist_fetch_address(transports).await;
+            if !hosts
+                .container
+                .fetch_address(HostColor::Grey, transports, transport_mixing)
+                .await
+                .is_empty()
+            {
+                let addrs = hosts
+                    .container
+                    .fetch_address(HostColor::Grey, transports, transport_mixing)
+                    .await;
 
                 return hosts.check_address(addrs).await
             }
@@ -264,7 +314,7 @@ impl Slot {
 
             // Do peer discovery if we don't have a hostlist (first time connecting
             // to the network).
-            if hosts.is_empty_hostlist().await {
+            if hosts.container.is_empty(HostColor::Grey).await {
                 dnetev!(self, OutboundSlotSleeping, {
                     slot: self.slot,
                 });
@@ -365,7 +415,7 @@ impl Slot {
             self.channel_id.store(channel.info.id, Ordering::Relaxed);
 
             // Add this connection to the anchorlist
-            hosts.upgrade_host(&addr).await;
+            hosts.container.upgrade_host(&addr).await;
 
             // Wait for channel to close
             stop_sub.receive().await;

src/net/session/seedsync_session.rs

@@ -51,6 +51,7 @@ use url::Url;
 use super::{
     super::{
         connector::Connector,
+        hosts::store::HostColor,
         p2p::{P2p, P2pPtr},
     },
     Session, SessionBitFlag, SESSION_SEED,
@@ -115,7 +116,7 @@ impl SeedSyncSession {
         }
 
         // Seed process complete
-        if self.p2p().hosts().is_empty_greylist().await {
+        if self.p2p().hosts().container.is_empty(HostColor::Grey).await {
             warn!(target: "net::session::seedsync_session", "[P2P] Greylist empty after seeding");
         }
 

src/net/tests.rs

@@ -26,7 +26,7 @@ use smol::{channel, future, Executor};
 use url::Url;
 
 use crate::{
-    net::{P2p, Settings},
+    net::{hosts::store::HostColor, P2p, Settings},
     system::sleep,
 };
 
@@ -158,7 +158,7 @@ async fn hostlist_propagation(ex: Arc<Executor<'static>>) {
     for p2p in p2p_instances.iter() {
         let hosts = p2p.hosts();
         // We should have some greylist entries at this point.
-        assert!(!hosts.is_empty_greylist().await);
+        assert!(!hosts.container.is_empty(HostColor::Grey).await);
     }
 
     // Stop the P2P network