hosts.rs 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761
  1. /* This file is part of DarkFi (https://dark.fi)
  2. *
  3. * Copyright (C) 2020-2023 Dyne.org foundation
  4. *
  5. * This program is free software: you can redistribute it and/or modify
  6. * it under the terms of the GNU Affero General Public License as
  7. * published by the Free Software Foundation, either version 3 of the
  8. * License, or (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU Affero General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU Affero General Public License
  16. * along with this program. If not, see <https://www.gnu.org/licenses/>.
  17. */
  18. use std::{collections::HashSet, sync::Arc};
  19. use log::{debug, trace};
  20. use rand::{
  21. prelude::{IteratorRandom, SliceRandom},
  22. rngs::OsRng,
  23. };
  24. use smol::lock::RwLock;
  25. use url::Url;
  26. use super::{p2p::P2pPtr, settings::SettingsPtr};
  27. use crate::{
  28. system::{Subscriber, SubscriberPtr, Subscription},
  29. Error, Result,
  30. };
  31. /// Atomic pointer to hosts object
  32. pub type HostsPtr = Arc<Hosts>;
  33. // An array containing all possible local host strings
  34. // TODO: This could perhaps be more exhaustive?
  35. pub const LOCAL_HOST_STRS: [&str; 2] = ["localhost", "localhost.localdomain"];
  36. /// Manages a store of network addresses
  37. pub struct Hosts {
  38. // Intermediary node list that is periodically probed and updated to whitelist.
  39. pub greylist: RwLock<Vec<(Url, u64)>>,
  40. // Recently seen nodes.
  41. pub whitelist: RwLock<Vec<(Url, u64)>>,
  42. /// Peers we reject from connecting
  43. rejected: RwLock<HashSet<String>>,
  44. /// Subscriber listening for store updates
  45. store_subscriber: SubscriberPtr<usize>,
  46. /// Pointer to configured P2P settings
  47. settings: SettingsPtr,
  48. }
  49. impl Hosts {
  50. /// Create a new hosts list>
  51. pub fn new(settings: SettingsPtr) -> HostsPtr {
  52. Arc::new(Self {
  53. whitelist: RwLock::new(Vec::new()),
  54. greylist: RwLock::new(Vec::new()),
  55. rejected: RwLock::new(HashSet::new()),
  56. store_subscriber: Subscriber::new(),
  57. settings,
  58. })
  59. }
  60. /// Loops through whitelist addresses to find an outbound address that we can
  61. /// connect to. Check whether the address is valid by making sure it isn't
  62. /// our own inbound address, then checks whether it is already connected
  63. /// (exists) or connecting (pending).
  64. /// Lastly adds matching address to the pending list.
  65. pub async fn whitelist_fetch_address_with_lock(
  66. &self,
  67. p2p: P2pPtr,
  68. transports: &[String],
  69. ) -> Option<(Url, u64)> {
  70. // Collect hosts
  71. let mut hosts = vec![];
  72. // If transport mixing is enabled, then for example we're allowed to
  73. // use tor:// to connect to tcp:// and tor+tls:// to connect to tcp+tls://.
  74. // However, **do not** mix tor:// and tcp+tls://, nor tor+tls:// and tcp://.
  75. let transport_mixing = self.settings.transport_mixing;
  76. macro_rules! mix_transport {
  77. ($a:expr, $b:expr) => {
  78. if transports.contains(&$a.to_string()) && transport_mixing {
  79. let mut a_to_b =
  80. self.whitelist_fetch_with_schemes(&[$b.to_string()], None).await;
  81. for (addr, last_seen) in a_to_b.iter_mut() {
  82. addr.set_scheme($a).unwrap();
  83. hosts.push((addr.clone(), last_seen.clone()));
  84. }
  85. }
  86. };
  87. }
  88. mix_transport!("tor", "tcp");
  89. mix_transport!("tor+tls", "tcp+tls");
  90. mix_transport!("nym", "tcp");
  91. mix_transport!("nym+tls", "tcp+tls");
  92. // And now the actual requested transports
  93. for (addr, last_seen) in self.whitelist_fetch_with_schemes(transports, None).await {
  94. hosts.push((addr, last_seen));
  95. }
  96. // Randomize hosts list. Do not try to connect in a deterministic order.
  97. // This is healthier for multiple slots to not compete for the same addrs.
  98. hosts.shuffle(&mut OsRng);
  99. // Try to find an unused host in the set.
  100. for (host, last_seen) in hosts.iter() {
  101. // Check if we already have this connection established
  102. if p2p.exists(host).await {
  103. trace!(
  104. target: "net::hosts::whitelist_fetch_address_with_lock()",
  105. "Host '{}' exists so skipping",
  106. host
  107. );
  108. continue
  109. }
  110. // Check if we already have this configured as a manual peer
  111. if self.settings.peers.contains(host) {
  112. trace!(
  113. target: "net::hosts::whitelist_fetch_address_with_lock()",
  114. "Host '{}' configured as manual peer so skipping",
  115. host
  116. );
  117. continue
  118. }
  119. // Obtain a lock on this address to prevent duplicate connection
  120. if !p2p.add_pending(host).await {
  121. trace!(
  122. target: "net::hosts::whitelist_fetch_address_with_lock()",
  123. "Host '{}' pending so skipping",
  124. host
  125. );
  126. continue
  127. }
  128. trace!(
  129. target: "net::hosts::whitelist_fetch_address_with_lock()",
  130. "Found valid host '{}",
  131. host
  132. );
  133. return Some((host.clone(), last_seen.clone()))
  134. }
  135. None
  136. }
  137. // Store the address in the whitelist if we don't have it.
  138. // Otherwise, update the last_seen field.
  139. // TODO: test the performance of this method. It might be costly.
  140. pub async fn whitelist_store_or_update(&self, addr: &Url, last_seen: u64) -> Result<()> {
  141. debug!(target: "net::hosts::whitelist_store_or_update()",
  142. "hosts::whitelist_store_or_update() [START]");
  143. if !self.whitelist_contains(addr).await {
  144. self.whitelist_store(addr, last_seen).await;
  145. } else {
  146. let index = self.get_whitelist_index_at_addr(addr).await?;
  147. self.whitelist_update_last_seen(addr, last_seen, index).await;
  148. }
  149. Ok(())
  150. }
  151. // Update the last_seen field for a Url on the whitelist.
  152. pub async fn whitelist_update(&self, addr: &Url, last_seen: u64) -> Result<()> {
  153. let index = self.get_whitelist_index_at_addr(addr).await?;
  154. self.whitelist_update_last_seen(addr, last_seen, index).await;
  155. Ok(())
  156. }
  157. pub async fn greylist_store_or_update(&self, addrs: &[(Url, u64)]) -> Result<()> {
  158. debug!(target: "net::hosts::greylist_store_or_update()",
  159. "hosts::greylist_store_or_update() [START]");
  160. for (addr, last_seen) in addrs {
  161. if !self.greylist_contains(addr).await {
  162. debug!(target: "net::greylist_store_or_update()", "New greylist candidate found!");
  163. // TODO: clean this up: greylist_store one item at a time
  164. self.greylist_store(&[(addr.clone(), last_seen.clone())]).await;
  165. } else {
  166. debug!(target: "net::greylist_store_or_update()",
  167. "Existing greylist entry found. Updating last_seen...");
  168. let index = self.get_greylist_index_at_addr(addr).await?;
  169. self.greylist_update_last_seen(addr, last_seen.clone(), index).await;
  170. }
  171. }
  172. Ok(())
  173. }
  174. // Append host to the greylist. Called on learning of a new peer.
  175. pub async fn greylist_store(&self, addrs: &[(Url, u64)]) {
  176. debug!(target: "net::hosts::greylist_store()", "hosts::greylist_store() [START]");
  177. debug!(target: "net::hosts::greylist_store()", "Filtering addresses...");
  178. let filtered_addrs = self.filter_addresses(addrs).await;
  179. let filtered_addrs_len = filtered_addrs.len();
  180. debug!(target: "net::hosts::greylist_store()", "Filtered addresses.");
  181. if !filtered_addrs.is_empty() {
  182. debug!(target: "net::hosts::greylist_store()", "Starting greylist write...");
  183. let mut greylist = self.greylist.write().await;
  184. debug!(target: "net::hosts::greylist_store()", "Achieved write lock on greylist!");
  185. // Remove oldest element if the greylist reaches max size.
  186. if greylist.len() == 5000 {
  187. let last_entry = greylist.pop().unwrap();
  188. debug!(target: "net::hosts::greylist_store()", "Greylist reached max size. Removed {:?}", last_entry);
  189. } else {
  190. for (addr, last_seen) in filtered_addrs {
  191. debug!(target: "net::hosts::greylist_store()", "Inserting {}", addr);
  192. greylist.push((addr.clone(), last_seen.clone()))
  193. }
  194. // Sort the list by last_seen.
  195. greylist.sort_unstable_by_key(|entry| entry.1);
  196. debug!(target: "net::hosts::greylist_store()", "Sorted greylist: {:?}", greylist)
  197. }
  198. } else {
  199. debug!(target: "net::hosts::greylist_store()", "Empty address message...")
  200. }
  201. self.store_subscriber.notify(filtered_addrs_len).await;
  202. debug!(target: "net::hosts::greylist_store()", "hosts::greylist_store() [END]");
  203. }
  204. // Append host to the whitelist. Called after a successful interaction with an online peer.
  205. pub async fn whitelist_store(&self, addr: &Url, last_seen: u64) {
  206. debug!(target: "net::hosts::whitelist_store()", "hosts::whitelist_store() [START]");
  207. let mut whitelist = self.whitelist.write().await;
  208. debug!(target: "net::hosts::whitelist_store()", "Inserting {}. Last seen {:?}", addr, last_seen);
  209. // Remove oldest element if the whitelist reaches max size.
  210. if whitelist.len() == 1000 {
  211. let last_entry = whitelist.pop().unwrap();
  212. debug!(target: "net::hosts::whitelist_store()", "Whitelist reached max size. Removed {:?}", last_entry);
  213. }
  214. whitelist.push((addr.clone(), last_seen));
  215. // Sort the list by last_seen.
  216. whitelist.sort_unstable_by_key(|entry| entry.1);
  217. debug!(target: "net::hosts::whitelist_store()", "Sorted whitelist: {:?}", whitelist);
  218. debug!(target: "net::hosts::whitelist_store()", "hosts::whitelist_store() [END]");
  219. }
  220. // Update the last_seen field of a peer on the whitelist.
  221. pub async fn whitelist_update_last_seen(&self, addr: &Url, last_seen: u64, index: usize) {
  222. debug!(target: "net::hosts::update_last_seen()", "hosts::update_last_seen() [START]");
  223. let mut whitelist = self.whitelist.write().await;
  224. whitelist[index] = (addr.clone(), last_seen);
  225. }
  226. // Update the last_seen field of a peer on the greylist.
  227. pub async fn greylist_update_last_seen(&self, addr: &Url, last_seen: u64, index: usize) {
  228. debug!(target: "net::hosts::greylist_update_last_seen()",
  229. "hosts::greylist_update_last_seen() [START]");
  230. let mut greylist = self.greylist.write().await;
  231. greylist[index] = (addr.clone(), last_seen);
  232. }
  233. pub async fn whitelist_downgrade(&self, addr: &Url) {
  234. // First lookup the entry using its addr.
  235. let mut entry = vec![];
  236. let whitelist = self.whitelist.read().await;
  237. for (url, time) in whitelist.iter() {
  238. if url == addr {
  239. entry.push((url.clone(), time.clone()));
  240. }
  241. }
  242. // TODO: This is for testing purposes.
  243. assert!(entry.len() == 1);
  244. // Remove this item from the whitelist.
  245. let mut whitelist = self.whitelist.write().await;
  246. // TODO: test!
  247. let index = whitelist.iter().position(|x| *x == entry[0]);
  248. // This should never fail since the entry exists.
  249. whitelist.remove(index.unwrap());
  250. // Add it to the greylist.
  251. let addr = entry[0].0.clone();
  252. let last_seen = entry[0].1.clone();
  253. self.greylist_store(&[(addr, last_seen)]).await;
  254. }
  255. pub async fn subscribe_store(&self) -> Result<Subscription<usize>> {
  256. let sub = self.store_subscriber.clone().subscribe().await;
  257. Ok(sub)
  258. }
  259. // Verify whether a URL is local.
  260. // NOTE: This function is stateless and not specific to
  261. // `Hosts`. For this reason, it might make more sense
  262. // to move this function to a more appropriate location
  263. // in the codebase.
  264. pub async fn is_local_host(&self, url: Url) -> bool {
  265. // Reject Urls without host strings.
  266. if url.host_str().is_none() {
  267. return false
  268. }
  269. // We do this hack in order to parse IPs properly.
  270. // https://github.com/whatwg/url/issues/749
  271. let addr = Url::parse(&url.as_str().replace(url.scheme(), "http")).unwrap();
  272. // Filter private IP ranges
  273. match addr.host().unwrap() {
  274. url::Host::Ipv4(ip) => {
  275. if !ip.is_global() {
  276. return true
  277. }
  278. }
  279. url::Host::Ipv6(ip) => {
  280. if !ip.is_global() {
  281. return true
  282. }
  283. }
  284. url::Host::Domain(d) => {
  285. if LOCAL_HOST_STRS.contains(&d) {
  286. return true
  287. }
  288. }
  289. }
  290. false
  291. }
  292. /// Filter given addresses based on certain rulesets and validity.
  293. async fn filter_addresses(&self, addrs: &[(Url, u64)]) -> Vec<(Url, u64)> {
  294. debug!(target: "net::hosts::filter_addresses()", "Filtering addrs: {:?}", addrs);
  295. let mut ret = vec![];
  296. let localnet = self.settings.localnet;
  297. 'addr_loop: for (addr_, last_seen) in addrs {
  298. // Validate that the format is `scheme://host_str:port`
  299. if addr_.host_str().is_none() ||
  300. addr_.port().is_none() ||
  301. addr_.cannot_be_a_base() ||
  302. addr_.path_segments().is_some()
  303. {
  304. continue
  305. }
  306. if self.is_rejected(addr_).await {
  307. debug!(target: "net::hosts::filter_addresses()", "Peer {} is rejected", addr_);
  308. continue
  309. }
  310. let host_str = addr_.host_str().unwrap();
  311. if !localnet {
  312. // Our own external addresses should never enter the hosts set.
  313. for ext in &self.settings.external_addrs {
  314. if host_str == ext.host_str().unwrap() {
  315. continue 'addr_loop
  316. }
  317. }
  318. }
  319. // We do this hack in order to parse IPs properly.
  320. // https://github.com/whatwg/url/issues/749
  321. let addr = Url::parse(&addr_.as_str().replace(addr_.scheme(), "http")).unwrap();
  322. // Filter non-global ranges if we're not allowing localnet.
  323. // Should never be allowed in production, so we don't really care
  324. // about some of them (e.g. 0.0.0.0, or broadcast, etc.).
  325. if !localnet && self.is_local_host(addr).await {
  326. continue
  327. }
  328. match addr_.scheme() {
  329. // Validate that the address is an actual onion.
  330. #[cfg(feature = "p2p-tor")]
  331. "tor" | "tor+tls" => {
  332. use std::str::FromStr;
  333. if tor_hscrypto::pk::HsId::from_str(host_str).is_err() {
  334. continue
  335. }
  336. debug!(target: "net::hosts::filter_addresses()", "[Tor] Valid: {}", host_str);
  337. }
  338. #[cfg(feature = "p2p-nym")]
  339. "nym" | "nym+tls" => continue, // <-- Temp skip
  340. #[cfg(feature = "p2p-tcp")]
  341. "tcp" | "tcp+tls" => {
  342. debug!(target: "net::hosts::filter_addresses()", "[TCP] Valid: {}", host_str);
  343. }
  344. _ => continue,
  345. }
  346. ret.push((addr_.clone(), last_seen.clone()));
  347. }
  348. ret
  349. }
  350. /// Check if a given peer (URL) is in the set of rejected hosts
  351. pub async fn is_rejected(&self, peer: &Url) -> bool {
  352. // Skip lookup for UNIX sockets and localhost connections
  353. // as they should never belong to the list of rejected URLs.
  354. let Some(hostname) = peer.host_str() else { return false };
  355. if self.is_local_host(peer.clone()).await {
  356. return false
  357. }
  358. self.rejected.read().await.contains(hostname)
  359. }
  360. /// Mark a peer as rejected by adding it to the set of rejected URLs.
  361. pub async fn mark_rejected(&self, peer: &Url) {
  362. // We ignore UNIX sockets here so we will just work
  363. // with stuff that has host_str().
  364. if let Some(hostname) = peer.host_str() {
  365. // Localhost connections should not be rejected
  366. // This however allows any Tor and Nym connections.
  367. if self.is_local_host(peer.clone()).await {
  368. return
  369. }
  370. self.rejected.write().await.insert(hostname.to_string());
  371. }
  372. }
  373. /// Unmark a rejected peer
  374. pub async fn unmark_rejected(&self, peer: &Url) {
  375. if let Some(hostname) = peer.host_str() {
  376. self.rejected.write().await.remove(hostname);
  377. }
  378. }
  379. /// Check if the greylist is empty.
  380. pub async fn is_empty_greylist(&self) -> bool {
  381. self.greylist.read().await.is_empty()
  382. }
  383. /// Check if the whitelist is empty.
  384. pub async fn is_empty_whitelist(&self) -> bool {
  385. self.whitelist.read().await.is_empty()
  386. }
  387. /// Check if host is in the greylist
  388. pub async fn greylist_contains(&self, addr: &Url) -> bool {
  389. let greylist = self.greylist.read().await;
  390. if greylist.iter().any(|(u, _t)| u == addr) {
  391. return true
  392. }
  393. return false
  394. }
  395. /// Check if host is in the whitelist
  396. pub async fn whitelist_contains(&self, addr: &Url) -> bool {
  397. let whitelist = self.whitelist.read().await;
  398. if whitelist.iter().any(|(u, _t)| u == addr) {
  399. return true
  400. }
  401. return false
  402. }
  403. /// Get the index for a given addr on the whitelist.
  404. pub async fn get_whitelist_index_at_addr(&self, addr: &Url) -> Result<(usize)> {
  405. let whitelist = self.whitelist.read().await;
  406. for (i, (url, _time)) in whitelist.iter().enumerate() {
  407. if url == addr {
  408. return Ok(i)
  409. }
  410. }
  411. return Err(Error::InvalidIndex)
  412. }
  413. /// Get the index for a given addr on the greylist.
  414. pub async fn get_greylist_index_at_addr(&self, addr: &Url) -> Result<(usize)> {
  415. let greylist = self.greylist.read().await;
  416. for (i, (url, _time)) in greylist.iter().enumerate() {
  417. if url == addr {
  418. return Ok(i)
  419. }
  420. }
  421. return Err(Error::InvalidIndex)
  422. }
  423. /// Return all known whitelisted hosts
  424. pub async fn whitelist_fetch_all(&self) -> Vec<(Url, u64)> {
  425. self.whitelist.read().await.iter().cloned().collect()
  426. }
  427. /// Get up to n random peers from the whitelist.
  428. pub async fn fetch_n_random(&self, n: u32) -> Vec<(Url, u64)> {
  429. let n = n as usize;
  430. if n == 0 {
  431. return vec![]
  432. }
  433. let addrs = self.whitelist.read().await;
  434. let urls = addrs.iter().choose_multiple(&mut OsRng, n.min(addrs.len()));
  435. urls.iter().map(|&url| url.clone()).collect()
  436. }
  437. /// Get up to n random whitelisted peers that match the given transport schemes from the hosts set.
  438. pub async fn whitelist_fetch_n_random_with_schemes(
  439. &self,
  440. schemes: &[String],
  441. n: u32,
  442. ) -> Vec<(Url, u64)> {
  443. let n = n as usize;
  444. if n == 0 {
  445. return vec![]
  446. }
  447. // Retrieve all peers corresponding to that transport schemes
  448. let hosts = self.whitelist_fetch_with_schemes(schemes, None).await;
  449. if hosts.is_empty() {
  450. return hosts
  451. }
  452. // Grab random ones
  453. let urls = hosts.iter().choose_multiple(&mut OsRng, n.min(hosts.len()));
  454. urls.iter().map(|&url| url.clone()).collect()
  455. }
  456. /// Get up to n random whitelisted peers that don't match the given transport schemes from the hosts set.
  457. pub async fn whitelist_fetch_n_random_excluding_schemes(
  458. &self,
  459. schemes: &[String],
  460. n: u32,
  461. ) -> Vec<(Url, u64)> {
  462. let n = n as usize;
  463. if n == 0 {
  464. return vec![]
  465. }
  466. // Retrieve all peers not corresponding to that transport schemes
  467. let hosts = self.whitelist_fetch_excluding_schemes(schemes, None).await;
  468. if hosts.is_empty() {
  469. return hosts
  470. }
  471. // Grab random ones
  472. let urls = hosts.iter().choose_multiple(&mut OsRng, n.min(hosts.len()));
  473. urls.iter().map(|&url| url.clone()).collect()
  474. }
  475. /// Get up to limit peers that match the given transport schemes from the whitelist.
  476. /// If limit was not provided, return all matching peers.
  477. pub async fn whitelist_fetch_with_schemes(
  478. &self,
  479. schemes: &[String],
  480. limit: Option<usize>,
  481. ) -> Vec<(Url, u64)> {
  482. let whitelist = self.whitelist.read().await;
  483. let mut limit = match limit {
  484. Some(l) => l.min(whitelist.len()),
  485. None => whitelist.len(),
  486. };
  487. let mut ret = vec![];
  488. if limit == 0 {
  489. return ret
  490. }
  491. for (addr, last_seen) in whitelist.iter() {
  492. if schemes.contains(&addr.scheme().to_string()) {
  493. ret.push((addr.clone(), *last_seen));
  494. limit -= 1;
  495. if limit == 0 {
  496. return ret
  497. }
  498. }
  499. }
  500. // If we didn't find any, pick some from the greylist
  501. if ret.is_empty() {
  502. for (addr, last_seen) in self.greylist.read().await.iter() {
  503. if schemes.contains(&addr.scheme().to_string()) {
  504. ret.push((addr.clone(), *last_seen));
  505. limit -= 1;
  506. if limit == 0 {
  507. break
  508. }
  509. }
  510. }
  511. }
  512. ret
  513. }
  514. /// Get up to limit peers that don't match the given transport schemes from the whitelist.
  515. /// If limit was not provided, return all matching peers.
  516. pub async fn whitelist_fetch_excluding_schemes(
  517. &self,
  518. schemes: &[String],
  519. limit: Option<usize>,
  520. ) -> Vec<(Url, u64)> {
  521. let addrs = self.whitelist.read().await;
  522. let mut limit = match limit {
  523. Some(l) => l.min(addrs.len()),
  524. None => addrs.len(),
  525. };
  526. let mut ret = vec![];
  527. if limit == 0 {
  528. return ret
  529. }
  530. for (addr, last_seen) in addrs.iter() {
  531. if !schemes.contains(&addr.scheme().to_string()) {
  532. ret.push((addr.clone(), *last_seen));
  533. limit -= 1;
  534. if limit == 0 {
  535. return ret
  536. }
  537. }
  538. }
  539. // If we didn't find any, pick some from the greylist
  540. if ret.is_empty() {
  541. for (addr, last_seen) in self.greylist.read().await.iter() {
  542. if !schemes.contains(&addr.scheme().to_string()) {
  543. ret.push((addr.clone(), *last_seen));
  544. limit -= 1;
  545. if limit == 0 {
  546. break
  547. }
  548. }
  549. }
  550. }
  551. ret
  552. }
  553. }
  554. #[cfg(test)]
  555. mod tests {
  556. use super::{super::settings::Settings, *};
  557. use std::time::UNIX_EPOCH;
  558. #[test]
  559. fn test_is_local_host() {
  560. smol::block_on(async {
  561. let settings = Settings {
  562. localnet: false,
  563. external_addrs: vec![
  564. Url::parse("tcp://foo.bar:123").unwrap(),
  565. Url::parse("tcp://lol.cat:321").unwrap(),
  566. ],
  567. ..Default::default()
  568. };
  569. let hosts = Hosts::new(Arc::new(settings.clone()));
  570. let local_hosts: Vec<Url> = vec![
  571. Url::parse("tcp://localhost").unwrap(),
  572. Url::parse("tcp://127.0.0.1").unwrap(),
  573. Url::parse("tcp+tls://[::1]").unwrap(),
  574. Url::parse("tcp://localhost.localdomain").unwrap(),
  575. Url::parse("tcp://192.168.10.65").unwrap(),
  576. ];
  577. for host in local_hosts {
  578. eprintln!("{}", host);
  579. assert!(hosts.is_local_host(host).await);
  580. }
  581. let remote_hosts: Vec<Url> = vec![
  582. Url::parse("https://dyne.org").unwrap(),
  583. Url::parse("tcp://77.168.10.65:2222").unwrap(),
  584. Url::parse("tcp://[2345:0425:2CA1:0000:0000:0567:5673:23b5]").unwrap(),
  585. Url::parse("http://eweiibe6tdjsdprb4px6rqrzzcsi22m4koia44kc5pcjr7nec2rlxyad.onion")
  586. .unwrap(),
  587. ];
  588. for host in remote_hosts {
  589. assert!(!(hosts.is_local_host(host).await))
  590. }
  591. });
  592. }
  593. #[test]
  594. fn test_greylist_store() {
  595. let last_seen = UNIX_EPOCH.elapsed().unwrap().as_secs();
  596. smol::block_on(async {
  597. let settings = Settings {
  598. localnet: false,
  599. external_addrs: vec![
  600. Url::parse("tcp://foo.bar:123").unwrap(),
  601. Url::parse("tcp://lol.cat:321").unwrap(),
  602. ],
  603. ..Default::default()
  604. };
  605. let hosts = Hosts::new(Arc::new(settings.clone()));
  606. let mut external_addrs = vec![];
  607. for addr in settings.external_addrs {
  608. external_addrs.push((addr, last_seen))
  609. }
  610. hosts.greylist_store(&external_addrs).await;
  611. assert!(hosts.is_empty_greylist().await);
  612. let local_hosts = vec![
  613. (Url::parse("tcp://localhost:3921").unwrap(), last_seen),
  614. (Url::parse("tor://[::1]:21481").unwrap(), last_seen),
  615. (Url::parse("tcp://192.168.10.65:311").unwrap(), last_seen),
  616. (Url::parse("tcp+tls://0.0.0.0:2312").unwrap(), last_seen),
  617. (Url::parse("tcp://255.255.255.255:2131").unwrap(), last_seen),
  618. ];
  619. hosts.greylist_store(&local_hosts).await;
  620. assert!(hosts.is_empty_greylist().await);
  621. let remote_hosts = vec![
  622. (Url::parse("tcp://dark.fi:80").unwrap(), last_seen),
  623. (Url::parse("tcp://http.cat:401").unwrap(), last_seen),
  624. (Url::parse("tcp://foo.bar:111").unwrap(), last_seen),
  625. ];
  626. hosts.greylist_store(&remote_hosts).await;
  627. assert!(hosts.greylist_contains(&remote_hosts[0].0).await);
  628. assert!(hosts.greylist_contains(&remote_hosts[1].0).await);
  629. assert!(!hosts.greylist_contains(&remote_hosts[2].0).await);
  630. });
  631. }
  632. #[test]
  633. fn test_whitelist_store() {
  634. smol::block_on(async {
  635. let settings = Settings {
  636. localnet: false,
  637. external_addrs: vec![
  638. Url::parse("tcp://foo.bar:123").unwrap(),
  639. Url::parse("tcp://lol.cat:321").unwrap(),
  640. ],
  641. ..Default::default()
  642. };
  643. let hosts = Hosts::new(Arc::new(settings.clone()));
  644. assert!(hosts.is_empty_whitelist().await);
  645. let url = Url::parse("tcp://dark.renaissance:333").unwrap();
  646. let last_seen = UNIX_EPOCH.elapsed().unwrap().as_secs();
  647. hosts.whitelist_store(&url, last_seen).await;
  648. assert!(!hosts.is_empty_whitelist().await);
  649. assert!(hosts.whitelist_contains(&url).await);
  650. });
  651. }
  652. }