store: cleanup

organize methods better, fix debug statements and comments, remove
hostlist_fetch_safe method.

src/net/hosts/store.rs

@@ -39,6 +39,12 @@ use crate::{
     Error, Result,
 };
 
+// An array containing all possible local host strings
+// TODO: This could perhaps be more exhaustive?
+pub const LOCAL_HOST_STRS: [&str; 2] = ["localhost", "localhost.localdomain"];
+const WHITELIST_MAX_LEN: usize = 5000;
+const GREYLIST_MAX_LEN: usize = 2000;
+
 /// Atomic pointer to hosts object
 pub type HostsPtr = Arc<Hosts>;
 
@@ -146,432 +152,337 @@ pub enum HostColor {
     Black = 3,
 }
 
-/// A Container for managing Grey, White, Gold and Black hostlists. Exposes a common interface for
-/// writing to and querying hostlists.
-/// TODO: Currently hosts (aside from hosts on the Black list) are on multiple lists at once.
-/// This needs to be considered closely.
+/// A Container for managing Grey, White, Gold and Black
+/// hostlists. Exposes a common interface for writing to and querying
+/// hostlists.
+// TODO: Currently hosts (aside from hosts on the Black list) are on
+// multiple lists at once. This needs to be reconsidered.
+// Rethink upgrade/ downgrade methods and consider a single method move() which
+// removes from one hostlist and places on another.
 // TODO: Verify the performance overhead of using vectors for hostlists.
 // TODO: Check whether anchorlist (Gold) has a max size in Monero.
 pub struct HostContainer {
     pub hostlists: [RwLock<Vec<(Url, u64)>>; 4],
 }
 
-// An array containing all possible local host strings
-// TODO: This could perhaps be more exhaustive?
-pub const LOCAL_HOST_STRS: [&str; 2] = ["localhost", "localhost.localdomain"];
-const WHITELIST_MAX_LEN: usize = 5000;
-const GREYLIST_MAX_LEN: usize = 2000;
-
-/// Manages a store of network addresses
-pub struct Hosts {
-    /// Subscriber for notifications of new channels
-    channel_subscriber: SubscriberPtr<Result<ChannelPtr>>,
-
-    /// Set of stored addresses that are quarantined.
-    /// We quarantine peers we've been unable to connect to, but we keep them
-    /// around so we can potentially try them again, up to n tries. This should
-    /// be helpful in order to self-heal the p2p connections in case we have an
-    /// Internet interrupt (goblins unplugging cables)
-    quarantine: RwLock<HashMap<Url, usize>>,
-
-    /// A registry that tracks hosts and their current state.
-    registry: HostRegistry,
-
-    /// Subscriber listening for store updates
-    store_subscriber: SubscriberPtr<usize>,
-
-    /// Pointer to configured P2P settings
-    settings: SettingsPtr,
-
-    pub container: HostContainer,
-}
+impl HostContainer {
+    fn new() -> Self {
+        let hostlists: [RwLock<Vec<(Url, u64)>>; 4] = [
+            RwLock::new(Vec::new()),
+            RwLock::new(Vec::new()),
+            RwLock::new(Vec::new()),
+            RwLock::new(Vec::new()),
+        ];
 
-impl Hosts {
-    /// Create a new hosts list>
-    pub fn new(settings: SettingsPtr) -> HostsPtr {
-        Arc::new(Self {
-            channel_subscriber: Subscriber::new(),
-            quarantine: RwLock::new(HashMap::new()),
-            registry: RwLock::new(HashMap::new()),
-            store_subscriber: Subscriber::new(),
-            settings,
-            container: HostContainer::new(),
-        })
+        Self { hostlists }
     }
 
-    /// Safely insert into the HostContainer. Filters the addresses first before storing and
-    /// notifies the subscriber. Must be called when first receiving greylist addresses.
-    pub async fn insert(&self, color: HostColor, addrs: &[(Url, u64)]) {
-        trace!(target: "net::hosts:insert()", "[START]");
-        let filtered_addrs = self.filter_addresses(self.settings.clone(), addrs).await;
-        let filtered_addrs_len = filtered_addrs.len();
+    /// Append host to a hostlist.
+    pub async fn store(&self, color: usize, addr: Url, last_seen: u64) {
+        trace!(target: "net::hosts::store()", "[START]");
+        let mut list = self.hostlists[color].write().await;
 
-        if filtered_addrs.is_empty() {
-            debug!(target: "net::hosts::insert()", "Filtered out all addresses");
+        list.push((addr, last_seen));
+
+        if color == 0 {
+            if list.len() == GREYLIST_MAX_LEN {
+                let last_entry = list.pop().unwrap();
+                debug!(target: "net::hosts::store()",
+            "Greylist reached max size. Removed {:?}", last_entry);
+            }
+        }
+        if color == 1 {
+            if list.len() == WHITELIST_MAX_LEN {
+                let last_entry = list.pop().unwrap();
+                debug!(target: "net::hosts::store()",
+            "Whitelist reached max size. Removed {:?}", last_entry);
+            }
         }
 
-        self.container.store_or_update(color, &filtered_addrs).await;
-        self.store_subscriber.notify(filtered_addrs_len).await;
+        // Sort the list by last_seen.
+        list.sort_by_key(|entry| entry.1);
+        list.reverse();
+        trace!(target: "net::hosts::store()", "[END]");
     }
 
-    /// Try to update the registry. If the host already exists, try to update its state.
-    /// Otherwise add the host to the registry along with its state.
-    pub async fn try_register(&self, addr: Url, new_state: HostState) -> Result<HostState> {
-        let mut registry = self.registry.write().await;
-
-        if registry.contains_key(&addr) {
-            let current_state = registry.get(&addr).unwrap().clone();
-
-            debug!(target: "net::hosts::store::try_update_registry()",
-            "Attempting to update addr={} current_state={}, new_state={}",
-            addr, current_state, new_state.to_string());
+    /// Stores an address on a hostlist or updates its last_seen field if we already
+    /// have the address.
+    pub async fn store_or_update(&self, color: HostColor, addrs: &[(Url, u64)]) {
+        trace!(target: "net::hosts::store_or_update()", "[START]");
+        let parent_index = color as usize;
+        for (addr, last_seen) in addrs {
+            if !self.contains(parent_index, &addr).await {
+                debug!(target: "net::hosts::store_or_update()",
+                    "We do not have this entry in the hostlist. Adding to store...");
 
-            let result: Result<HostState> = match new_state {
-                HostState::Pending => current_state.try_pending(),
-                HostState::Connected(c) => current_state.try_connect(c),
-                HostState::Downgrading => current_state.try_downgrade(),
-                HostState::Refining => current_state.try_refine(),
-            };
+                self.store(parent_index, addr.clone(), *last_seen).await;
+            } else {
+                debug!(target: "net::hosts::store_or_update()",
+                        "We have this entry in the hostlist. Updating last seen...");
 
-            if let Ok(state) = &result {
-                registry.insert(addr.clone(), state.clone());
+                let child_index = self
+                    .get_index_at_addr(parent_index, addr.clone())
+                    .await
+                    .expect("Expected entry to exist");
+                debug!(target: "net::hosts::store_or_update()",
+                        "Selected index, updating last seen...");
+                self.update_last_seen(parent_index, &addr, *last_seen, child_index).await;
             }
-
-            result
-        } else {
-            // We don't know this peer. We can safely update the state.
-            registry.insert(addr.clone(), new_state.clone());
-
-            Ok(new_state)
         }
     }
 
-    pub async fn check_address(&self, hosts: Vec<(Url, u64)>) -> Option<(Url, u64)> {
-        // Try to find an unused host in the set.
-        for (host, last_seen) in hosts {
-            debug!(target: "net::hosts::store::check_address()", "Starting checks");
+    /// Update the last_seen field of a peer on a hostlist.
+    pub async fn update_last_seen(&self, color: usize, addr: &Url, last_seen: u64, index: usize) {
+        trace!(target: "net::hosts::update_last_seen()", "[START]");
+        let mut list = self.hostlists[color].write().await;
 
-            if let Err(_) = self.try_register(host.clone(), HostState::Pending).await {
-                continue
-            }
+        list[index] = (addr.clone(), last_seen);
 
-            debug!(
-                target: "net::hosts::store::check_address()",
-                "Found valid host {}",
-                host
-            );
-            return Some((host.clone(), last_seen))
-        }
+        list.sort_by_key(|entry| entry.1);
+        list.reverse();
+        trace!(target: "net::hosts::update_last_seen()", "[END]");
+    }
 
-        None
+    /// Return all known hosts on a hostlist.
+    pub async fn fetch_all(&self, color: HostColor) -> Vec<(Url, u64)> {
+        self.hostlists[color as usize].read().await.iter().cloned().collect()
     }
 
-    /// Remove a host from the HostRegistry. Must be called after downgrade(), when the refinery
-    /// process fails, or when a channel stops. Prevents hosts from getting trapped in the
-    /// HostState logical machinery.
-    pub async fn unregister(&self, addr: &Url) {
-        debug!(target: "net::hosts::store::unregister()", "Removing {} from HostRegistry", addr);
-        self.registry.write().await.remove(addr);
+    /// Get the oldest entry from a hostlist.
+    pub async fn fetch_last(&self, color: HostColor) -> ((Url, u64), usize) {
+        let list = self.hostlists[color as usize].read().await;
+        let position = list.len() - 1;
+        let entry = &list[position];
+        (entry.clone(), position)
     }
 
-    /// Returns the list of connected channels.
-    pub async fn channels(&self) -> Vec<ChannelPtr> {
-        let registry = self.registry.read().await;
-        let mut channels = Vec::new();
+    /// TODO: documentation
+    pub async fn fetch_address(
+        &self,
+        color: HostColor,
+        transports: &[String],
+        transport_mixing: bool,
+    ) -> Vec<(Url, u64)> {
+        trace!(target: "net::hosts::fetch_address()", "[START]");
+        let mut hosts = vec![];
+        let index = color as usize;
 
-        for (_, value) in registry.iter() {
-            if let HostState::Connected(c) = value {
-                channels.push(c.clone());
-            }
+        // If transport mixing is enabled, then for example we're allowed to
+        // use tor:// to connect to tcp:// and tor+tls:// to connect to tcp+tls://.
+        // However, **do not** mix tor:// and tcp+tls://, nor tor+tls:// and tcp://.
+        macro_rules! mix_transport {
+            ($a:expr, $b:expr) => {
+                if transports.contains(&$a.to_string()) && transport_mixing {
+                    let mut a_to_b = self.fetch_with_schemes(index, &[$b.to_string()], None).await;
+                    for (addr, last_seen) in a_to_b.iter_mut() {
+                        addr.set_scheme($a).unwrap();
+                        hosts.push((addr.clone(), last_seen.clone()));
+                    }
+                }
+            };
         }
-        channels
-    }
-
-    /// Retrieve a random connected channel
-    pub async fn random_channel(&self) -> ChannelPtr {
-        let channels = self.channels().await;
-        let position = rand::thread_rng().gen_range(0..channels.len());
-        channels[position].clone()
-    }
 
-    /// Add a channel to the set of connected channels
-    pub async fn register_channel(&self, channel: ChannelPtr) -> Result<()> {
-        let address = channel.address().clone();
+        mix_transport!("tor", "tcp");
+        mix_transport!("tor+tls", "tcp+tls");
+        mix_transport!("nym", "tcp");
+        mix_transport!("nym+tls", "tcp+tls");
 
-        if let Err(e) =
-            self.try_register(address.clone(), HostState::Connected(channel.clone())).await
-        {
-            return Err(e)
+        // And now the actual requested transports
+        for (addr, last_seen) in self.fetch_with_schemes(index, transports, None).await {
+            hosts.push((addr, last_seen));
         }
 
-        self.channel_subscriber.notify(Ok(channel)).await;
-        Ok(())
-    }
+        trace!(target: "net::hosts::fetch_address()", "Grabbed hosts, length: {}", hosts.len());
 
-    pub async fn subscribe_store(&self) -> Result<Subscription<usize>> {
-        let sub = self.store_subscriber.clone().subscribe().await;
-        Ok(sub)
+        hosts
     }
 
-    // Verify whether a URL is local.
-    // NOTE: This function is stateless and not specific to
-    // `Hosts`. For this reason, it might make more sense
-    // to move this function to a more appropriate location
-    // in the codebase.
-    /// Check whether a URL is local host
-    pub async fn is_local_host(&self, url: Url) -> bool {
-        // Reject Urls without host strings.
-        if url.host_str().is_none() {
-            return false
-        }
+    /// Get up to limit peers that match the given transport schemes from a hostlist.
+    /// If limit was not provided, return all matching peers.
+    async fn fetch_with_schemes(
+        &self,
+        color: usize,
+        schemes: &[String],
+        limit: Option<usize>,
+    ) -> Vec<(Url, u64)> {
+        trace!(target: "net::hosts::fetch_with_schemes()", "[START]");
+        let list = self.hostlists[color].read().await;
 
-        // We do this hack in order to parse IPs properly.
-        // https://github.com/whatwg/url/issues/749
-        let addr = Url::parse(&url.as_str().replace(url.scheme(), "http")).unwrap();
-        // Filter private IP ranges
-        match addr.host().unwrap() {
-            url::Host::Ipv4(ip) => {
-                if !ip.is_global() {
-                    return true
-                }
-            }
-            url::Host::Ipv6(ip) => {
-                if !ip.is_global() {
-                    return true
-                }
-            }
-            url::Host::Domain(d) => {
-                if LOCAL_HOST_STRS.contains(&d) {
-                    return true
+        let mut limit = match limit {
+            Some(l) => l.min(list.len()),
+            None => list.len(),
+        };
+        let mut ret = vec![];
+
+        if limit == 0 {
+            return ret
+        }
+
+        for (addr, last_seen) in list.iter() {
+            if schemes.contains(&addr.scheme().to_string()) {
+                ret.push((addr.clone(), *last_seen));
+                limit -= 1;
+                if limit == 0 {
+                    debug!(target: "net::hosts::fetch_with_schemes()",
+                        "Found matching scheme, returning {} addresses",
+                        ret.len());
+                    return ret
                 }
             }
         }
-        false
+
+        if ret.is_empty() {
+            debug!(target: "net::hosts::fetch_with_schemes()",
+                  "No such schemes found!")
+        }
+
+        ret
     }
 
-    /// Filter given addresses based on certain rulesets and validity.
-    async fn filter_addresses(
+    /// Get up to limit peers that don't match the given transport schemes from a hostlist.
+    /// If limit was not provided, return all matching peers.
+    pub async fn fetch_excluding_schemes(
         &self,
-        settings: SettingsPtr,
-        addrs: &[(Url, u64)],
+        color: usize,
+        schemes: &[String],
+        limit: Option<usize>,
     ) -> Vec<(Url, u64)> {
-        trace!(target: "net::hosts::store::filter_addresses()", "Filtering addrs: {:?}", addrs);
-        let mut ret = vec![];
-        let localnet = self.settings.localnet;
-
-        'addr_loop: for (addr_, last_seen) in addrs {
-            // Validate that the format is `scheme://host_str:port`
-            if addr_.host_str().is_none() ||
-                addr_.port().is_none() ||
-                addr_.cannot_be_a_base() ||
-                addr_.path_segments().is_some()
-            {
-                continue
-            }
+        let list = self.hostlists[color].read().await;
 
-            if self.container.contains(HostColor::Black as usize, addr_).await {
-                warn!(target: "net::hosts::filter_addresses()",
-                "Peer {} is blacklisted", addr_);
-                continue
-            }
+        let mut limit = match limit {
+            Some(l) => l.min(list.len()),
+            None => list.len(),
+        };
+        let mut ret = vec![];
 
-            let host_str = addr_.host_str().unwrap();
+        if limit == 0 {
+            return ret
+        }
 
-            if !localnet {
-                // Our own external addresses should never enter the hosts set.
-                for ext in &settings.external_addrs {
-                    if host_str == ext.host_str().unwrap() {
-                        continue 'addr_loop
-                    }
-                }
-            }
-            // On localnet, make sure ours ports don't enter the host set.
-            for ext in &settings.external_addrs {
-                if addr_.port() == ext.port() {
-                    continue 'addr_loop
+        for (addr, last_seen) in list.iter() {
+            if !schemes.contains(&addr.scheme().to_string()) {
+                ret.push((addr.clone(), *last_seen));
+                limit -= 1;
+                if limit == 0 {
+                    return ret
                 }
             }
+        }
 
-            // We do this hack in order to parse IPs properly.
-            // https://github.com/whatwg/url/issues/749
-            let addr = Url::parse(&addr_.as_str().replace(addr_.scheme(), "http")).unwrap();
-
-            // Filter non-global ranges if we're not allowing localnet.
-            // Should never be allowed in production, so we don't really care
-            // about some of them (e.g. 0.0.0.0, or broadcast, etc.).
-            if !localnet && self.is_local_host(addr).await {
-                continue
-            }
-
-            match addr_.scheme() {
-                // Validate that the address is an actual onion.
-                #[cfg(feature = "p2p-tor")]
-                "tor" | "tor+tls" => {
-                    use std::str::FromStr;
-                    if tor_hscrypto::pk::HsId::from_str(host_str).is_err() {
-                        continue
-                    }
-                    trace!(target: "net::hosts::store::filter_addresses()",
-                    "[Tor] Valid: {}", host_str);
-                }
+        if ret.is_empty() {
+            debug!(target: "net::hosts::fetch_excluding_schemes()",
+                    "No such schemes found!")
+        }
 
-                #[cfg(feature = "p2p-nym")]
-                "nym" | "nym+tls" => continue, // <-- Temp skip
+        ret
+    }
 
-                #[cfg(feature = "p2p-tcp")]
-                "tcp" | "tcp+tls" => {
-                    trace!(target: "net::hosts::store::filter_addresses()",
-                    "[TCP] Valid: {}", host_str);
-                }
+    /// Get a random peer from a hostlist.
+    pub async fn fetch_random(&self, color: HostColor) -> ((Url, u64), usize) {
+        let list = self.hostlists[color as usize].read().await;
+        let position = rand::thread_rng().gen_range(0..list.len());
+        let entry = &list[position];
+        (entry.clone(), position)
+    }
 
-                _ => continue,
-            }
+    /// Get a random peer from a hostlist that matches the given transport schemes.
+    pub async fn fetch_random_with_schemes(
+        &self,
+        color: HostColor,
+        schemes: &[String],
+    ) -> Option<((Url, u64), usize)> {
+        // Retrieve all peers corresponding to that transport schemes
+        trace!(target: "net::hosts::fetch_random_with_schemes()", "[START]");
+        let list = self.fetch_with_schemes(color as usize, schemes, None).await;
 
-            ret.push((addr_.clone(), *last_seen));
+        if list.is_empty() {
+            return None
         }
 
-        ret
+        let position = rand::thread_rng().gen_range(0..list.len());
+        let entry = &list[position];
+        Some((entry.clone(), position))
     }
-    /// Downgrade a host to greylist. If the host is on the anchorlist or whitelist, remove it.
-    /// If it's already on the greylist we can't do anything here.
-    pub async fn downgrade_host(&self, addr: &Url, last_seen: u64) {
-        if let Err(_) = self.try_register(addr.clone(), HostState::Downgrading).await {
-            return
-        }
 
-        debug!(target: "net::hosts::store::downgrade_host()", "Downgrading host {}", addr);
-        if self.container.contains(HostColor::Grey as usize, addr).await {
-            warn!(target: "net::hosts::downgrade_host()",
-                  "Cannot downgrade a host that is already on the greylist! {}", addr);
+    /// Get up to n random peers. Schemes are not taken into account.
+    pub async fn fetch_n_random(&self, color: HostColor, n: u32) -> Vec<(Url, u64)> {
+        trace!(target: "net::hosts::fetch_n_random()", "[START]");
+        let n = n as usize;
+        if n == 0 {
+            return vec![]
         }
+        let mut hosts = vec![];
 
-        if self.container.contains(HostColor::Gold as usize, addr).await {
-            debug!(target: "net::hosts::downgrade_host()", "Removing from anchorlist {}", addr);
-
-            let index = self
-                .container
-                .get_index_at_addr(HostColor::Gold as usize, addr.clone())
-                .await
-                .expect("Expected anchorlist index to exist");
+        let list = self.hostlists[color as usize].read().await;
 
-            self.container.remove(HostColor::Gold, addr, index).await;
-            self.container.store_or_update(HostColor::Grey, &[(addr.clone(), last_seen)]).await;
+        for (addr, last_seen) in list.iter() {
+            hosts.push((addr.clone(), *last_seen));
         }
 
-        if self.container.contains(HostColor::White as usize, addr).await {
-            debug!(target: "net::hosts::downgrade_host()", "Removing from whitelist {}", addr);
-
-            let index = self
-                .container
-                .get_index_at_addr(HostColor::White as usize, addr.clone())
-                .await
-                .expect("Expected whitelist index to exist");
-
-            self.container.remove(HostColor::White, addr, index).await;
-            self.container.store_or_update(HostColor::Grey, &[(addr.clone(), last_seen)]).await;
+        if hosts.is_empty() {
+            debug!(target: "net::hosts::fetch_n_random()",
+                        "No entries found!");
+            return hosts
         }
 
-        // Remove this entry from HostRegistry to avoid this host getting
-        // stuck in the Downgrading state.
-        self.unregister(&addr).await;
+        // Grab random ones
+        let urls = hosts.iter().choose_multiple(&mut OsRng, n.min(hosts.len()));
+        urls.iter().map(|&url| url.clone()).collect()
     }
 
-    /// Quarantine a peer.
-    /// If they've been quarantined for more than a configured limit, downgrade to greylist.
-    pub async fn quarantine(&self, addr: &Url, last_seen: u64) {
-        debug!(target: "net::hosts::store::quarantine()", "Quarantining peer {}", addr);
-        let timer = Instant::now();
-        let mut q = self.quarantine.write().await;
-        if let Some(retries) = q.get_mut(addr) {
-            *retries += 1;
-            debug!(target: "net::hosts::store::quarantine()",
-            "Peer {} quarantined {} times", addr, retries);
-            if *retries == self.settings.hosts_quarantine_limit {
-                debug!(target: "net::hosts::store::quarantine()",
-                "Reached quarantine limited after {:?}", timer.elapsed());
-                debug!(target: "net::hosts::store::quarantine()",
-                "Removing from hostlist {}", addr);
-                drop(q);
-                self.downgrade_host(addr, last_seen).await;
-            }
-        } else {
-            debug!(target: "net::hosts::quarantine()", "Added peer {} to quarantine", addr);
-            q.insert(addr.clone(), 0);
+    /// Get up to n random peers that match the given transport schemes.
+    pub async fn fetch_n_random_with_schemes(
+        &self,
+        color: HostColor,
+        schemes: &[String],
+        n: u32,
+    ) -> Vec<(Url, u64)> {
+        trace!(target: "net::hosts::fetch_n_random_with_schemes()", "[START]");
+        let index = color as usize;
+        let n = n as usize;
+        if n == 0 {
+            return vec![]
         }
-    }
 
-    /// Mark a peer as blacklist.
-    pub async fn blacklist(&self, peer: &Url) {
-        // We ignore UNIX sockets here so we will just work
-        // with stuff that has host_str().
-        if let Some(_) = peer.host_str() {
-            // Localhost connections should never enter the blacklist
-            // This however allows any Tor and Nym connections.
-            if self.is_local_host(peer.clone()).await {
-                return
-            }
-
-            // Insert into the blacklist. We set last_seen to 0 (we don't care about this
-            // field).
-            self.container.hostlists[HostColor::Black as usize]
-                .write()
-                .await
-                .push((peer.clone(), 0));
+        // Retrieve all peers corresponding to that transport schemes
+        let hosts = self.fetch_with_schemes(index, schemes, None).await;
+        if hosts.is_empty() {
+            debug!(target: "net::hosts::fetch_n_random_with_schemes()",
+                  "No such schemes found!");
+            return hosts
         }
-    }
-}
 
-impl HostContainer {
-    fn new() -> Self {
-        let hostlists: [RwLock<Vec<(Url, u64)>>; 4] = [
-            RwLock::new(Vec::new()),
-            RwLock::new(Vec::new()),
-            RwLock::new(Vec::new()),
-            RwLock::new(Vec::new()),
-        ];
-
-        Self { hostlists }
+        // Grab random ones
+        let urls = hosts.iter().choose_multiple(&mut OsRng, n.min(hosts.len()));
+        urls.iter().map(|&url| url.clone()).collect()
     }
 
-    /// TODO: documentation
-    pub async fn fetch_address(
+    /// Get up to n random peers that don't match the given transport schemes from
+    /// a hostlist.
+    pub async fn fetch_n_random_excluding_schemes(
         &self,
         color: HostColor,
-        transports: &[String],
-        transport_mixing: bool,
+        schemes: &[String],
+        n: u32,
     ) -> Vec<(Url, u64)> {
-        trace!(target: "net::hosts::fetch_address()", "[START]");
-        let mut hosts = vec![];
+        trace!(target: "net::hosts::fetch_excluding_schemes()", "[START]");
         let index = color as usize;
-
-        // If transport mixing is enabled, then for example we're allowed to
-        // use tor:// to connect to tcp:// and tor+tls:// to connect to tcp+tls://.
-        // However, **do not** mix tor:// and tcp+tls://, nor tor+tls:// and tcp://.
-        macro_rules! mix_transport {
-            ($a:expr, $b:expr) => {
-                if transports.contains(&$a.to_string()) && transport_mixing {
-                    let mut a_to_b = self.fetch_with_schemes(index, &[$b.to_string()], None).await;
-                    for (addr, last_seen) in a_to_b.iter_mut() {
-                        addr.set_scheme($a).unwrap();
-                        hosts.push((addr.clone(), last_seen.clone()));
-                    }
-                }
-            };
+        let n = n as usize;
+        if n == 0 {
+            return vec![]
         }
+        // Retrieve all peers not corresponding to that transport schemes
+        let hosts = self.fetch_excluding_schemes(index, schemes, None).await;
 
-        mix_transport!("tor", "tcp");
-        mix_transport!("tor+tls", "tcp+tls");
-        mix_transport!("nym", "tcp");
-        mix_transport!("nym+tls", "tcp+tls");
-
-        // And now the actual requested transports
-        for (addr, last_seen) in self.fetch_with_schemes(index, transports, None).await {
-            hosts.push((addr, last_seen));
+        if hosts.is_empty() {
+            debug!(target: "net::hosts::fetch_n_random_excluding_schemes()",
+            "No such schemes found!");
+            return hosts
         }
 
-        trace!(target: "net::hosts::fetch_address()", "Grabbed hosts, length: {}", hosts.len());
-
-        hosts
+        // Grab random ones
+        let urls = hosts.iter().choose_multiple(&mut OsRng, n.min(hosts.len()));
+        urls.iter().map(|&url| url.clone()).collect()
     }
 
     /// Upgrade a connection to the anchorlist. Called after a connection has been successfully
@@ -581,412 +492,493 @@ impl HostContainer {
         self.store_or_update(HostColor::Gold, &[(addr.clone(), last_seen)]).await;
     }
 
-    /// Stores an address on a hostlist or updates its last_seen field if we already
-    /// have the address.
-    pub async fn store_or_update(&self, color: HostColor, addrs: &[(Url, u64)]) {
-        trace!(target: "net::hosts::store_or_update()", "[START]");
-        let parent_index = color as usize;
-        for (addr, last_seen) in addrs {
-            if !self.contains(parent_index, &addr).await {
-                debug!(target: "net::hosts::store_or_update()",
-                    "We do not have this entry in the hostlist. Adding to store...");
+    /// Remove an entry from a hostlist.
+    pub async fn remove(&self, color: HostColor, addr: &Url, index: usize) {
+        debug!(target: "net::hosts::remove()", "Removing peer {} from hostlist", addr);
+        let mut list = self.hostlists[color as usize].write().await;
+        list.remove(index);
+    }
 
-                self.store(parent_index, addr.clone(), *last_seen).await;
-            } else {
-                debug!(target: "net::hosts::store_or_update()",
-                        "We have this entry in the hostlist. Updating last seen...");
+    /// Check if a hostlist is empty.
+    pub async fn is_empty(&self, color: HostColor) -> bool {
+        self.hostlists[color as usize].read().await.is_empty()
+    }
 
-                let child_index = self
-                    .get_index_at_addr(parent_index, addr.clone())
-                    .await
-                    .expect("Expected entry to exist");
-                debug!(target: "net::hosts::store_or_update()",
-                        "Selected index, updating last seen...");
-                self.update_last_seen(parent_index, &addr, *last_seen, child_index).await;
+    /// Check if host is in a hostlist
+    pub async fn contains(&self, color: usize, addr: &Url) -> bool {
+        self.hostlists[color].read().await.iter().any(|(u, _t)| u == addr)
+    }
+
+    /// Get the index for a given addr on a hostlist.
+    pub async fn get_index_at_addr(&self, color: usize, addr: Url) -> Option<usize> {
+        self.hostlists[color].read().await.iter().position(|a| a.0 == addr)
+    }
+
+    /// Get the entry for a given addr on the hostlist.
+    pub async fn get_entry_at_addr(&self, color: usize, addr: &Url) -> Option<(Url, u64)> {
+        self.hostlists[color]
+            .read()
+            .await
+            .iter()
+            .find(|(url, _)| url == addr)
+            .map(|(url, time)| (url.clone(), *time))
+    }
+
+    /// Load the hostlists from a file.
+    pub async fn load_all(&self, path: &String) -> Result<()> {
+        let path = expand_path(path)?;
+
+        if !path.exists() {
+            if let Some(parent) = path.parent() {
+                fs::create_dir_all(parent)?;
+            }
+
+            File::create(path.clone())?;
+        }
+
+        let contents = load_file(&path);
+        if let Err(e) = contents {
+            warn!(target: "net::hosts::load_hosts()", "Failed retrieving saved hosts: {}", e);
+            return Ok(())
+        }
+
+        for line in contents.unwrap().lines() {
+            let data: Vec<&str> = line.split('\t').collect();
+
+            let url = match Url::parse(data[1]) {
+                Ok(u) => u,
+                Err(e) => {
+                    debug!(target: "net::hosts::load_hosts()", "Skipping malformed URL {}", e);
+                    continue
+                }
+            };
+
+            let last_seen = match data[2].parse::<u64>() {
+                Ok(t) => t,
+                Err(e) => {
+                    debug!(target: "net::hosts::load_hosts()", "Skipping malformed last seen {}", e);
+                    continue
+                }
+            };
+
+            match data[0] {
+                "greylist" => {
+                    self.store(HostColor::Grey as usize, url, last_seen).await;
+                }
+                "whitelist" => {
+                    self.store(HostColor::White as usize, url, last_seen).await;
+                }
+                "anchorlist" => {
+                    self.store(HostColor::Gold as usize, url, last_seen).await;
+                }
+                _ => {
+                    debug!(target: "net::hosts::load_hosts()", "Malformed list name...");
+                }
             }
         }
+
+        Ok(())
     }
 
-    /// Append host to a hostlist.
-    pub async fn store(&self, color: usize, addr: Url, last_seen: u64) {
-        trace!(target: "net::hosts::store()", "[START]");
-        let mut list = self.hostlists[color].write().await;
+    /// Save the hostlist to a file. Whitelist gets written to the greylist to force
+    /// whitelist entries through the refinery on start.
+    pub async fn save_all(&self, path: &String) -> Result<()> {
+        let path = expand_path(path)?;
 
-        list.push((addr, last_seen));
+        let mut tsv = String::new();
+        let mut white = vec![];
+        let mut greygold: HashMap<String, Vec<(Url, u64)>> = HashMap::new();
 
-        if color == 0 {
-            if list.len() == GREYLIST_MAX_LEN {
-                let last_entry = list.pop().unwrap();
-                debug!(target: "net::hosts::store()",
-            "Greylist reached max size. Removed {:?}", last_entry);
+        // First gather all the whitelist entries we don't have in greylist.
+        for (url, last_seen) in self.fetch_all(HostColor::White).await {
+            if !self.contains(HostColor::Grey as usize, &url).await {
+                white.push((url, last_seen))
             }
         }
-        if color == 1 {
-            if list.len() == WHITELIST_MAX_LEN {
-                let last_entry = list.pop().unwrap();
-                debug!(target: "net::hosts::store()",
-            "Whitelist reached max size. Removed {:?}", last_entry);
+
+        // Then gather the greylist and anchorlist entries.
+        greygold.insert("anchorlist".to_string(), self.fetch_all(HostColor::Gold).await);
+        greygold.insert("greylist".to_string(), self.fetch_all(HostColor::Grey).await);
+
+        // We write whitelist entries to the greylist on p2p.stop() to force
+        // them through the refinery on start().
+        for (name, mut list) in greygold {
+            if name == *"greylist".to_string() {
+                list.append(&mut white)
+            }
+            for (url, last_seen) in list {
+                tsv.push_str(&format!("{}\t{}\t{}\n", name, url, last_seen));
             }
         }
 
-        // Sort the list by last_seen.
-        list.sort_by_key(|entry| entry.1);
-        list.reverse();
-        trace!(target: "net::hosts::store()", "[END]");
+        if !tsv.eq("") {
+            info!(target: "net::hosts::save_hosts()", "Saving hosts to: {:?}",
+                  path);
+            if let Err(e) = save_file(&path, &tsv) {
+                error!(target: "net::hosts::save_hosts()", "Failed saving hosts: {}", e);
+            }
+        }
+
+        Ok(())
     }
+}
+
+/// TODO: documentation
+pub struct Hosts {
+    /// Subscriber for notifications of new channels
+    channel_subscriber: SubscriberPtr<Result<ChannelPtr>>,
+
+    /// Set of stored addresses that are quarantined.
+    /// We quarantine peers we've been unable to connect to, but we keep them
+    /// around so we can potentially try them again, up to n tries. This should
+    /// be helpful in order to self-heal the p2p connections in case we have an
+    /// Internet interrupt (goblins unplugging cables)
+    quarantine: RwLock<HashMap<Url, usize>>,
+
+    /// A registry that tracks hosts and their current state.
+    registry: HostRegistry,
+
+    /// Subscriber listening for store updates
+    store_subscriber: SubscriberPtr<usize>,
 
-    /// Update the last_seen field of a peer on a hostlist.
-    pub async fn update_last_seen(&self, color: usize, addr: &Url, last_seen: u64, index: usize) {
-        trace!(target: "net::hosts::update_last_seen()", "[START]");
-        let mut list = self.hostlists[color].write().await;
+    /// Pointer to configured P2P settings
+    settings: SettingsPtr,
 
-        list[index] = (addr.clone(), last_seen);
+    pub container: HostContainer,
+}
 
-        list.sort_by_key(|entry| entry.1);
-        list.reverse();
-        trace!(target: "net::hosts::update_last_seen()", "[END]");
+impl Hosts {
+    /// Create a new hosts list>
+    pub fn new(settings: SettingsPtr) -> HostsPtr {
+        Arc::new(Self {
+            channel_subscriber: Subscriber::new(),
+            quarantine: RwLock::new(HashMap::new()),
+            registry: RwLock::new(HashMap::new()),
+            store_subscriber: Subscriber::new(),
+            settings,
+            container: HostContainer::new(),
+        })
     }
 
-    /// Remove an entry from a hostlist.
-    pub async fn remove(&self, color: HostColor, addr: &Url, index: usize) {
-        debug!(target: "net::hosts::remove()", "Removing peer {} from hostlist", addr);
-        let mut list = self.hostlists[color as usize].write().await;
-        list.remove(index);
-    }
+    /// Safely insert into the HostContainer. Filters the addresses first before storing and
+    /// notifies the subscriber. Must be called when first receiving greylist addresses.
+    pub async fn insert(&self, color: HostColor, addrs: &[(Url, u64)]) {
+        trace!(target: "net::hosts:insert()", "[START]");
+        let filtered_addrs = self.filter_addresses(self.settings.clone(), addrs).await;
+        let filtered_addrs_len = filtered_addrs.len();
 
-    /// Check if a hostlist is empty.
-    pub async fn is_empty(&self, color: HostColor) -> bool {
-        self.hostlists[color as usize].read().await.is_empty()
-    }
+        if filtered_addrs.is_empty() {
+            debug!(target: "net::hosts::insert()", "Filtered out all addresses");
+        }
 
-    /// Check if host is in a hostlist
-    pub async fn contains(&self, color: usize, addr: &Url) -> bool {
-        self.hostlists[color].read().await.iter().any(|(u, _t)| u == addr)
+        self.container.store_or_update(color, &filtered_addrs).await;
+        self.store_subscriber.notify(filtered_addrs_len).await;
     }
 
-    /// Get the index for a given addr on a hostlist.
-    pub async fn get_index_at_addr(&self, color: usize, addr: Url) -> Option<usize> {
-        self.hostlists[color].read().await.iter().position(|a| a.0 == addr)
-    }
+    /// Try to update the registry. If the host already exists, try to update its state.
+    /// Otherwise add the host to the registry along with its state.
+    pub async fn try_register(&self, addr: Url, new_state: HostState) -> Result<HostState> {
+        let mut registry = self.registry.write().await;
 
-    /// Get the entry for a given addr on the hostlist.
-    pub async fn get_entry_at_addr(&self, color: usize, addr: &Url) -> Option<(Url, u64)> {
-        self.hostlists[color]
-            .read()
-            .await
-            .iter()
-            .find(|(url, _)| url == addr)
-            .map(|(url, time)| (url.clone(), *time))
-    }
+        if registry.contains_key(&addr) {
+            let current_state = registry.get(&addr).unwrap().clone();
 
-    /// Return all known hosts on a hostlist.
-    pub async fn fetch_all(&self, color: HostColor) -> Vec<(Url, u64)> {
-        self.hostlists[color as usize].read().await.iter().cloned().collect()
-    }
+            debug!(target: "net::hosts::try_update_registry()",
+            "Attempting to update addr={} current_state={}, new_state={}",
+            addr, current_state, new_state.to_string());
 
-    /// Return all greylist and anchorlist hosts. Called on stop().
-    /// Note: we do not return whitelist entries here since whitelist entries must go via the
-    /// greylist refinery in the lifetime of the p2p network.
-    pub async fn hostlist_fetch_safe(&self) -> HashMap<String, Vec<(Url, u64)>> {
-        let mut hostlist = HashMap::new();
-        hostlist.insert(
-            "anchorlist".to_string(),
-            self.hostlists[HostColor::Gold as usize].read().await.iter().cloned().collect(),
-        );
-        hostlist.insert(
-            "greylist".to_string(),
-            self.hostlists[HostColor::Grey as usize].read().await.iter().cloned().collect(),
-        );
-        hostlist
-    }
+            let result: Result<HostState> = match new_state {
+                HostState::Pending => current_state.try_pending(),
+                HostState::Connected(c) => current_state.try_connect(c),
+                HostState::Downgrading => current_state.try_downgrade(),
+                HostState::Refining => current_state.try_refine(),
+            };
 
-    /// Get a random peer from a hostlist.
-    pub async fn fetch_random(&self, color: HostColor) -> ((Url, u64), usize) {
-        let list = self.hostlists[color as usize].read().await;
-        let position = rand::thread_rng().gen_range(0..list.len());
-        let entry = &list[position];
-        (entry.clone(), position)
-    }
+            if let Ok(state) = &result {
+                registry.insert(addr.clone(), state.clone());
+            }
 
-    /// Get the oldest entry from a hostlist.
-    pub async fn fetch_last(&self, color: HostColor) -> ((Url, u64), usize) {
-        let list = self.hostlists[color as usize].read().await;
-        let position = list.len() - 1;
-        let entry = &list[position];
-        (entry.clone(), position)
+            result
+        } else {
+            // We don't know this peer. We can safely update the state.
+            registry.insert(addr.clone(), new_state.clone());
+
+            Ok(new_state)
+        }
     }
 
-    /// Get a random peer from a hostlist that matches the given transport schemes.
-    pub async fn fetch_random_with_schemes(
-        &self,
-        color: HostColor,
-        schemes: &[String],
-    ) -> Option<((Url, u64), usize)> {
-        trace!(target: "net::hosts::fetch_random_with_schemes()", "[START]");
-        // Retrieve all peers corresponding to that transport schemes
-        let list = self.fetch_with_schemes(color as usize, schemes, None).await;
+    pub async fn check_address(&self, hosts: Vec<(Url, u64)>) -> Option<(Url, u64)> {
+        // Try to find an unused host in the set.
+        for (host, last_seen) in hosts {
+            debug!(target: "net::hosts::check_address()", "Starting checks");
 
-        if list.is_empty() {
-            return None
+            if let Err(_) = self.try_register(host.clone(), HostState::Pending).await {
+                continue
+            }
+
+            debug!(
+                target: "net::hosts::check_address()",
+                "Found valid host {}",
+                host
+            );
+            return Some((host.clone(), last_seen))
         }
 
-        let position = rand::thread_rng().gen_range(0..list.len());
-        let entry = &list[position];
-        Some((entry.clone(), position))
+        None
     }
 
-    /// Get up to n random peers. Schemes are not taken into account.
-    pub async fn fetch_n_random(&self, color: HostColor, n: u32) -> Vec<(Url, u64)> {
-        trace!(target: "net::hosts::fetch_n_random()", "[START]");
-        let n = n as usize;
-        if n == 0 {
-            return vec![]
-        }
-        let mut hosts = vec![];
-
-        let list = self.hostlists[color as usize].read().await;
+    /// Remove a host from the HostRegistry. Must be called after downgrade(), when the refinery
+    /// process fails, or when a channel stops. Prevents hosts from getting trapped in the
+    /// HostState logical machinery.
+    pub async fn unregister(&self, addr: &Url) {
+        debug!(target: "net::hosts::unregister()", "Removing {} from HostRegistry", addr);
+        self.registry.write().await.remove(addr);
+    }
 
-        for (addr, last_seen) in list.iter() {
-            hosts.push((addr.clone(), *last_seen));
-        }
+    /// Returns the list of connected channels.
+    pub async fn channels(&self) -> Vec<ChannelPtr> {
+        let registry = self.registry.read().await;
+        let mut channels = Vec::new();
 
-        if hosts.is_empty() {
-            debug!(target: "net::hosts::fetch_n_random()",
-                        "No entries found!");
-            return hosts
+        for (_, value) in registry.iter() {
+            if let HostState::Connected(c) = value {
+                channels.push(c.clone());
+            }
         }
+        channels
+    }
 
-        // Grab random ones
-        let urls = hosts.iter().choose_multiple(&mut OsRng, n.min(hosts.len()));
-        urls.iter().map(|&url| url.clone()).collect()
+    /// Retrieve a random connected channel
+    pub async fn random_channel(&self) -> ChannelPtr {
+        let channels = self.channels().await;
+        let position = rand::thread_rng().gen_range(0..channels.len());
+        channels[position].clone()
     }
 
-    /// Get up to n random peers that match the given transport schemes.
-    pub async fn fetch_n_random_with_schemes(
-        &self,
-        color: HostColor,
-        schemes: &[String],
-        n: u32,
-    ) -> Vec<(Url, u64)> {
-        trace!(target: "net::hosts::fetch_n_random_with_schemes()", "[START]");
-        let index = color as usize;
-        let n = n as usize;
-        if n == 0 {
-            return vec![]
-        }
+    /// Add a channel to the set of connected channels
+    pub async fn register_channel(&self, channel: ChannelPtr) -> Result<()> {
+        let address = channel.address().clone();
 
-        // Retrieve all peers corresponding to that transport schemes
-        let hosts = self.fetch_with_schemes(index, schemes, None).await;
-        if hosts.is_empty() {
-            debug!(target: "net::hosts::fetch_n_random_with_schemes()",
-                  "No such schemes found!");
-            return hosts
+        if let Err(e) =
+            self.try_register(address.clone(), HostState::Connected(channel.clone())).await
+        {
+            return Err(e)
         }
 
-        // Grab random ones
-        let urls = hosts.iter().choose_multiple(&mut OsRng, n.min(hosts.len()));
-        urls.iter().map(|&url| url.clone()).collect()
+        self.channel_subscriber.notify(Ok(channel)).await;
+        Ok(())
     }
 
-    /// Get up to limit peers that don't match the given transport schemes from a hostlist.
-    /// If limit was not provided, return all matching peers.
-    pub async fn fetch_excluding_schemes(
-        &self,
-        color: usize,
-        schemes: &[String],
-        limit: Option<usize>,
-    ) -> Vec<(Url, u64)> {
-        let list = self.hostlists[color].read().await;
-
-        let mut limit = match limit {
-            Some(l) => l.min(list.len()),
-            None => list.len(),
-        };
-        let mut ret = vec![];
+    pub async fn subscribe_store(&self) -> Result<Subscription<usize>> {
+        let sub = self.store_subscriber.clone().subscribe().await;
+        Ok(sub)
+    }
 
-        if limit == 0 {
-            return ret
+    // Verify whether a URL is local.
+    // NOTE: This function is stateless and not specific to
+    // `Hosts`. For this reason, it might make more sense
+    // to move this function to a more appropriate location
+    // in the codebase.
+    /// Check whether a URL is local host
+    pub async fn is_local_host(&self, url: Url) -> bool {
+        // Reject Urls without host strings.
+        if url.host_str().is_none() {
+            return false
         }
 
-        for (addr, last_seen) in list.iter() {
-            if !schemes.contains(&addr.scheme().to_string()) {
-                ret.push((addr.clone(), *last_seen));
-                limit -= 1;
-                if limit == 0 {
-                    return ret
+        // We do this hack in order to parse IPs properly.
+        // https://github.com/whatwg/url/issues/749
+        let addr = Url::parse(&url.as_str().replace(url.scheme(), "http")).unwrap();
+        // Filter private IP ranges
+        match addr.host().unwrap() {
+            url::Host::Ipv4(ip) => {
+                if !ip.is_global() {
+                    return true
+                }
+            }
+            url::Host::Ipv6(ip) => {
+                if !ip.is_global() {
+                    return true
+                }
+            }
+            url::Host::Domain(d) => {
+                if LOCAL_HOST_STRS.contains(&d) {
+                    return true
                 }
             }
         }
-
-        if ret.is_empty() {
-            debug!(target: "net::hosts::fetch_excluding_schemes()",
-                    "No such schemes found!")
-        }
-
-        ret
+        false
     }
 
-    /// Get up to n random peers that don't match the given transport schemes from
-    /// a hostlist.
-    pub async fn fetch_n_random_excluding_schemes(
+    /// Filter given addresses based on certain rulesets and validity.
+    async fn filter_addresses(
         &self,
-        color: HostColor,
-        schemes: &[String],
-        n: u32,
+        settings: SettingsPtr,
+        addrs: &[(Url, u64)],
     ) -> Vec<(Url, u64)> {
-        trace!(target: "net::hosts::fetch_excluding_schemes()", "[START]");
-        let index = color as usize;
-        let n = n as usize;
-        if n == 0 {
-            return vec![]
-        }
-        // Retrieve all peers not corresponding to that transport schemes
-        let hosts = self.fetch_excluding_schemes(index, schemes, None).await;
+        trace!(target: "net::hosts::filter_addresses()", "Filtering addrs: {:?}", addrs);
+        let mut ret = vec![];
+        let localnet = self.settings.localnet;
 
-        if hosts.is_empty() {
-            debug!(target: "net::hosts::fetch_n_random_excluding_schemes()",
-            "No such schemes found!");
-            return hosts
-        }
+        'addr_loop: for (addr_, last_seen) in addrs {
+            // Validate that the format is `scheme://host_str:port`
+            if addr_.host_str().is_none() ||
+                addr_.port().is_none() ||
+                addr_.cannot_be_a_base() ||
+                addr_.path_segments().is_some()
+            {
+                continue
+            }
 
-        // Grab random ones
-        let urls = hosts.iter().choose_multiple(&mut OsRng, n.min(hosts.len()));
-        urls.iter().map(|&url| url.clone()).collect()
-    }
+            if self.container.contains(HostColor::Black as usize, addr_).await {
+                warn!(target: "net::hosts::filter_addresses()",
+                "Peer {} is blacklisted", addr_);
+                continue
+            }
 
-    /// Get up to limit peers that match the given transport schemes from a hostlist.
-    /// If limit was not provided, return all matching peers.
-    async fn fetch_with_schemes(
-        &self,
-        color: usize,
-        schemes: &[String],
-        limit: Option<usize>,
-    ) -> Vec<(Url, u64)> {
-        trace!(target: "net::hosts::fetch_with_schemes()", "[START]");
-        let list = self.hostlists[color].read().await;
+            let host_str = addr_.host_str().unwrap();
 
-        let mut limit = match limit {
-            Some(l) => l.min(list.len()),
-            None => list.len(),
-        };
-        let mut ret = vec![];
+            if !localnet {
+                // Our own external addresses should never enter the hosts set.
+                for ext in &settings.external_addrs {
+                    if host_str == ext.host_str().unwrap() {
+                        continue 'addr_loop
+                    }
+                }
+            }
+            // On localnet, make sure ours ports don't enter the host set.
+            for ext in &settings.external_addrs {
+                if addr_.port() == ext.port() {
+                    continue 'addr_loop
+                }
+            }
 
-        if limit == 0 {
-            return ret
-        }
+            // We do this hack in order to parse IPs properly.
+            // https://github.com/whatwg/url/issues/749
+            let addr = Url::parse(&addr_.as_str().replace(addr_.scheme(), "http")).unwrap();
 
-        for (addr, last_seen) in list.iter() {
-            if schemes.contains(&addr.scheme().to_string()) {
-                ret.push((addr.clone(), *last_seen));
-                limit -= 1;
-                if limit == 0 {
-                    debug!(target: "net::hosts::fetch_with_schemes()",
-                        "Found matching scheme, returning {} addresses",
-                        ret.len());
-                    return ret
-                }
+            // Filter non-global ranges if we're not allowing localnet.
+            // Should never be allowed in production, so we don't really care
+            // about some of them (e.g. 0.0.0.0, or broadcast, etc.).
+            if !localnet && self.is_local_host(addr).await {
+                continue
             }
-        }
 
-        if ret.is_empty() {
-            debug!(target: "net::hosts::fetch_with_schemes()",
-                  "No such schemes found!")
-        }
+            match addr_.scheme() {
+                // Validate that the address is an actual onion.
+                #[cfg(feature = "p2p-tor")]
+                "tor" | "tor+tls" => {
+                    use std::str::FromStr;
+                    if tor_hscrypto::pk::HsId::from_str(host_str).is_err() {
+                        continue
+                    }
+                    trace!(target: "net::hosts::filter_addresses()",
+                    "[Tor] Valid: {}", host_str);
+                }
 
-        ret
-    }
+                #[cfg(feature = "p2p-nym")]
+                "nym" | "nym+tls" => continue, // <-- Temp skip
 
-    /// Load the hostlists from a file.
-    pub async fn load_all(&self, path: &String) -> Result<()> {
-        let path = expand_path(path)?;
+                #[cfg(feature = "p2p-tcp")]
+                "tcp" | "tcp+tls" => {
+                    trace!(target: "net::hosts::filter_addresses()",
+                    "[TCP] Valid: {}", host_str);
+                }
 
-        if !path.exists() {
-            if let Some(parent) = path.parent() {
-                fs::create_dir_all(parent)?;
+                _ => continue,
             }
 
-            File::create(path.clone())?;
+            ret.push((addr_.clone(), *last_seen));
         }
 
-        let contents = load_file(&path);
-        if let Err(e) = contents {
-            warn!(target: "net::hosts::load_hosts()", "Failed retrieving saved hosts: {}", e);
-            return Ok(())
+        ret
+    }
+    /// Downgrade a host to greylist. If the host is on the anchorlist or whitelist, remove it.
+    /// If it's already on the greylist we can't do anything here.
+    pub async fn downgrade_host(&self, addr: &Url, last_seen: u64) {
+        if let Err(_) = self.try_register(addr.clone(), HostState::Downgrading).await {
+            return
         }
 
-        for line in contents.unwrap().lines() {
-            let data: Vec<&str> = line.split('\t').collect();
+        debug!(target: "net::hosts::downgrade_host()", "Downgrading host {}", addr);
+        if self.container.contains(HostColor::Grey as usize, addr).await {
+            warn!(target: "net::hosts::downgrade_host()",
+                  "Cannot downgrade a host that is already on the greylist! {}", addr);
+        }
 
-            let url = match Url::parse(data[1]) {
-                Ok(u) => u,
-                Err(e) => {
-                    debug!(target: "net::hosts::load_hosts()", "Skipping malformed URL {}", e);
-                    continue
-                }
-            };
+        if self.container.contains(HostColor::Gold as usize, addr).await {
+            debug!(target: "net::hosts::downgrade_host()", "Removing from anchorlist {}", addr);
 
-            let last_seen = match data[2].parse::<u64>() {
-                Ok(t) => t,
-                Err(e) => {
-                    debug!(target: "net::hosts::load_hosts()", "Skipping malformed last seen {}", e);
-                    continue
-                }
-            };
+            let index = self
+                .container
+                .get_index_at_addr(HostColor::Gold as usize, addr.clone())
+                .await
+                .expect("Expected anchorlist index to exist");
 
-            match data[0] {
-                "greylist" => {
-                    self.store(HostColor::Grey as usize, url, last_seen).await;
-                }
-                "whitelist" => {
-                    self.store(HostColor::White as usize, url, last_seen).await;
-                }
-                "anchorlist" => {
-                    self.store(HostColor::Gold as usize, url, last_seen).await;
-                }
-                _ => {
-                    debug!(target: "net::hosts::load_hosts()", "Malformed list name...");
-                }
-            }
+            self.container.remove(HostColor::Gold, addr, index).await;
+            self.container.store_or_update(HostColor::Grey, &[(addr.clone(), last_seen)]).await;
         }
 
-        Ok(())
-    }
-
-    /// Save the hostlist to a file. Whitelist gets written to the greylist to force
-    /// whitelist entries through the refinery on start.
-    pub async fn save_all(&self, path: &String) -> Result<()> {
-        let path = expand_path(path)?;
+        if self.container.contains(HostColor::White as usize, addr).await {
+            debug!(target: "net::hosts::downgrade_host()", "Removing from whitelist {}", addr);
 
-        let mut tsv = String::new();
-        let mut whitelist = vec![];
+            let index = self
+                .container
+                .get_index_at_addr(HostColor::White as usize, addr.clone())
+                .await
+                .expect("Expected whitelist index to exist");
 
-        // First gather all the whitelist entries we don't have in greylist.
-        for (url, last_seen) in self.fetch_all(HostColor::White).await {
-            if !self.contains(HostColor::Grey as usize, &url).await {
-                whitelist.push((url, last_seen))
-            }
+            self.container.remove(HostColor::White, addr, index).await;
+            self.container.store_or_update(HostColor::Grey, &[(addr.clone(), last_seen)]).await;
         }
 
-        // Collect the greylist and anchorlist entries, and append any whitelist entries to the
-        // greylist before saving.
-        for (name, mut list) in self.hostlist_fetch_safe().await {
-            if name == *"greylist".to_string() {
-                list.append(&mut whitelist)
-            }
-            for (url, last_seen) in list {
-                tsv.push_str(&format!("{}\t{}\t{}\n", name, url, last_seen));
+        // Remove this entry from HostRegistry to avoid this host getting
+        // stuck in the Downgrading state.
+        self.unregister(&addr).await;
+    }
+
+    /// Quarantine a peer.
+    /// If they've been quarantined for more than a configured limit, downgrade to greylist.
+    pub async fn quarantine(&self, addr: &Url, last_seen: u64) {
+        debug!(target: "net::hosts::quarantine()", "Quarantining peer {}", addr);
+        let timer = Instant::now();
+        let mut q = self.quarantine.write().await;
+        if let Some(retries) = q.get_mut(addr) {
+            *retries += 1;
+            debug!(target: "net::hosts::quarantine()",
+            "Peer {} quarantined {} times", addr, retries);
+            if *retries == self.settings.hosts_quarantine_limit {
+                debug!(target: "net::hosts::quarantine()",
+                "Reached quarantine limited after {:?}", timer.elapsed());
+                debug!(target: "net::hosts::quarantine()",
+                "Removing from hostlist {}", addr);
+                drop(q);
+                self.downgrade_host(addr, last_seen).await;
             }
+        } else {
+            debug!(target: "net::hosts::quarantine()", "Added peer {} to quarantine", addr);
+            q.insert(addr.clone(), 0);
         }
+    }
 
-        if !tsv.eq("") {
-            info!(target: "net::hosts::store::save_hosts()", "Saving hosts to: {:?}",
-                  path);
-            if let Err(e) = save_file(&path, &tsv) {
-                error!(target: "net::hosts::store::save_hosts()", "Failed saving hosts: {}", e);
+    /// Mark a peer as blacklist.
+    pub async fn blacklist(&self, peer: &Url) {
+        // We ignore UNIX sockets here so we will just work
+        // with stuff that has host_str().
+        if let Some(_) = peer.host_str() {
+            // Localhost connections should never enter the blacklist
+            // This however allows any Tor and Nym connections.
+            if self.is_local_host(peer.clone()).await {
+                return
             }
-        }
 
-        Ok(())
+            // Insert into the blacklist. We set last_seen to 0 (we don't care about this
+            // field).
+            self.container.hostlists[HostColor::Black as usize]
+                .write()
+                .await
+                .push((peer.clone(), 0));
+        }
     }
 }