Просмотр исходного кода

bin/ircd: remove buffers and sync messages

ghassmo 3 лет назад
Родитель
Сommit
b70f83303f
5 измененных файлов с 46 добавлено и 267 удалено
  1. 5 14
      bin/ircd/src/buffers.rs
  2. 10 14
      bin/ircd/src/irc/client.rs
  3. 9 6
      bin/ircd/src/irc/mod.rs
  4. 10 40
      bin/ircd/src/main.rs
  5. 12 193
      bin/ircd/src/protocol_privmsg.rs

+ 5 - 14
bin/ircd/src/buffers.rs

@@ -240,7 +240,7 @@ impl Orphan {
 }
 
 pub struct SeenIds {
-    ids: Mutex<RingBuffer<u64>>,
+    ids: RingBuffer<u64>,
 }
 
 impl Default for SeenIds {
@@ -251,13 +251,12 @@ impl Default for SeenIds {
 
 impl SeenIds {
     pub fn new() -> Self {
-        Self { ids: Mutex::new(RingBuffer::new(settings::SIZE_OF_IDSS_BUFFER)) }
+        Self { ids: RingBuffer::new(settings::SIZE_OF_IDSS_BUFFER) }
     }
 
-    pub async fn push(&self, id: u64) -> bool {
-        let ids = &mut self.ids.lock().await;
-        if !ids.contains(&id) {
-            ids.push(id);
+    pub fn push(&mut self, id: u64) -> bool {
+        if !self.ids.contains(&id) {
+            self.ids.push(id);
             return true
         }
         false
@@ -348,14 +347,6 @@ mod tests {
         assert_eq!(b.iter().last().unwrap(), &"h9");
     }
 
-    #[async_std::test]
-    async fn test_seen_ids() {
-        let seen_ids = SeenIds::default();
-        assert!(seen_ids.push(3000).await);
-        assert!(seen_ids.push(3001).await);
-        assert!(!seen_ids.push(3000).await);
-    }
-
     #[async_std::test]
     async fn test_unread_msgs() {
         let unread_msgs = UMsgs::default();

+ 10 - 14
bin/ircd/src/irc/client.rs

@@ -1,3 +1,4 @@
+use async_std::sync::{Arc, Mutex};
 use std::net::SocketAddr;
 
 use futures::{
@@ -14,7 +15,7 @@ use darkfi::{
 };
 
 use crate::{
-    buffers::Buffers,
+    buffers::SeenIds,
     crypto::{decrypt_privmsg, decrypt_target, encrypt_privmsg},
     settings,
     settings::RPL,
@@ -29,7 +30,7 @@ pub struct IrcClient<C: AsyncRead + AsyncWrite + Send + Unpin + 'static> {
     pub address: SocketAddr,
 
     // msgs buffer
-    buffers: Buffers,
+    seen: Arc<Mutex<SeenIds>>,
 
     // irc config
     irc_config: IrcConfig,
@@ -44,13 +45,13 @@ impl<C: AsyncRead + AsyncWrite + Send + Unpin + 'static> IrcClient<C> {
     pub fn new(
         write_stream: WriteHalf<C>,
         address: SocketAddr,
-        buffers: Buffers,
+        seen: Arc<Mutex<SeenIds>>,
         irc_config: IrcConfig,
         p2p: P2pPtr,
         notify_clients: SubscriberPtr<Privmsg>,
         subscription: Subscription<Privmsg>,
     ) -> Self {
-        Self { write_stream, address, buffers, irc_config, p2p, notify_clients, subscription }
+        Self { write_stream, address, seen, irc_config, p2p, notify_clients, subscription }
     }
 
     /// Start listening for messages came from p2p network or irc client
@@ -205,11 +206,6 @@ impl<C: AsyncRead + AsyncWrite + Send + Unpin + 'static> IrcClient<C> {
             if *self.irc_config.capabilities.get("no-history").unwrap() {
                 return Ok(())
             }
-
-            // Send dm messages in buffer
-            for msg in self.buffers.privmsgs.load().await {
-                self.process_msg(&msg).await?;
-            }
         }
         Ok(())
     }
@@ -446,9 +442,7 @@ impl<C: AsyncRead + AsyncWrite + Send + Unpin + 'static> IrcClient<C> {
 
         info!("[CLIENT {}] (Plain) PRIVMSG {} :{}", self.address, target, message,);
 
-        let last_term = self.buffers.privmsgs.last_term().await + 1;
-
-        let mut privmsg = Privmsg::new(&self.irc_config.nickname, target, &message, last_term);
+        let mut privmsg = Privmsg::new(&self.irc_config.nickname, target, &message, 0);
 
         if target.starts_with('#') {
             if !self.irc_config.configured_chans.contains_key(target) {
@@ -477,8 +471,10 @@ impl<C: AsyncRead + AsyncWrite + Send + Unpin + 'static> IrcClient<C> {
             }
         }
 
-        self.buffers.seen_ids.push(privmsg.id).await;
-        self.buffers.privmsgs.push(&privmsg).await;
+        {
+            let ids = &mut self.seen.lock().await;
+            ids.push(privmsg.id);
+        }
 
         self.notify_clients
             .notify_with_exclude(privmsg.clone(), &[self.subscription.get_id()])

+ 9 - 6
bin/ircd/src/irc/mod.rs

@@ -1,7 +1,10 @@
 use std::{fs::File, net::SocketAddr};
 
 use async_executor::Executor;
-use async_std::{net::TcpListener, sync::Arc};
+use async_std::{
+    net::TcpListener,
+    sync::{Arc, Mutex},
+};
 use futures::{io::BufReader, AsyncRead, AsyncReadExt, AsyncWrite};
 use futures_rustls::{rustls, TlsAcceptor};
 use fxhash::FxHashMap;
@@ -15,7 +18,7 @@ use darkfi::{
 };
 
 use crate::{
-    buffers::Buffers,
+    buffers::SeenIds,
     settings::{
         parse_configured_channels, parse_configured_contacts, Args, ChannelInfo, ContactInfo,
         CONFIG_FILE,
@@ -80,7 +83,7 @@ impl IrcConfig {
 pub struct IrcServer {
     settings: Args,
     irc_config: IrcConfig,
-    buffers: Buffers,
+    seen: Arc<Mutex<SeenIds>>,
     p2p: P2pPtr,
     notify_clients: SubscriberPtr<Privmsg>,
 }
@@ -88,12 +91,12 @@ pub struct IrcServer {
 impl IrcServer {
     pub async fn new(
         settings: Args,
-        buffers: Buffers,
+        seen: Arc<Mutex<SeenIds>>,
         p2p: P2pPtr,
         notify_clients: SubscriberPtr<Privmsg>,
     ) -> Result<Self> {
         let irc_config = IrcConfig::new(&settings)?;
-        Ok(Self { settings, irc_config, buffers, p2p, notify_clients })
+        Ok(Self { settings, irc_config, seen, p2p, notify_clients })
     }
 
     /// Start listening to new irc clients connecting to the irc server address
@@ -151,7 +154,7 @@ impl IrcServer {
         let mut client = IrcClient::new(
             writer,
             peer_addr,
-            self.buffers.clone(),
+            self.seen.clone(),
             self.irc_config.clone(),
             self.p2p.clone(),
             self.notify_clients.clone(),

+ 10 - 40
bin/ircd/src/main.rs

@@ -1,8 +1,7 @@
 use std::fmt;
-
 use async_channel::Receiver;
 use async_executor::Executor;
-use async_std::sync::Arc;
+use async_std::sync::{Arc, Mutex};
 
 use log::{info, warn};
 use rand::rngs::OsRng;
@@ -11,11 +10,9 @@ use structopt_toml::StructOptToml;
 
 use darkfi::{
     async_daemonize, net,
-    net::P2pPtr,
     rpc::server::listen_and_serve,
     system::{Subscriber, SubscriberPtr},
     util::{
-        async_util::sleep,
         cli::{get_log_config, get_log_level, spawn_config},
         file::save_json_file,
         path::{expand_path, get_config_path},
@@ -34,10 +31,10 @@ pub mod rpc;
 pub mod settings;
 
 use crate::{
-    buffers::{create_buffers, Buffers},
+    buffers::SeenIds,
     irc::IrcServer,
     privmsg::Privmsg,
-    protocol_privmsg::{LastTerm, ProtocolPrivmsg},
+    protocol_privmsg::ProtocolPrivmsg,
     rpc::JsonRpcInterface,
     settings::{Args, ChannelInfo, CONFIG_FILE, CONFIG_FILE_CONTENTS},
 };
@@ -54,25 +51,6 @@ impl fmt::Display for KeyPair {
     }
 }
 
-async fn resend_unread_msgs(p2p: P2pPtr, buffers: Buffers) -> Result<()> {
-    loop {
-        sleep(settings::TIMEOUT_FOR_RESEND_UNREAD_MSGS).await;
-
-        for msg in buffers.unread_msgs.load().await.values() {
-            p2p.broadcast(msg.clone()).await?;
-        }
-    }
-}
-
-async fn send_last_term(p2p: P2pPtr, buffers: Buffers) -> Result<()> {
-    loop {
-        sleep(settings::BROADCAST_LAST_TERM_MSG).await;
-
-        let term = buffers.privmsgs.last_term().await;
-        p2p.broadcast(LastTerm { term }).await?;
-    }
-}
-
 struct Ircd {
     notify_clients: SubscriberPtr<Privmsg>,
 }
@@ -86,7 +64,7 @@ impl Ircd {
     async fn start(
         &self,
         settings: &Args,
-        buffers: Buffers,
+        seen: Arc<Mutex<SeenIds>>,
         p2p: net::P2pPtr,
         p2p_receiver: Receiver<Privmsg>,
         executor: Arc<Executor<'_>>,
@@ -102,7 +80,7 @@ impl Ircd {
 
         let irc_server = IrcServer::new(
             settings.clone(),
-            buffers.clone(),
+            seen.clone(),
             p2p.clone(),
             self.notify_clients.clone(),
         )
@@ -120,7 +98,7 @@ impl Ircd {
 
 async_daemonize!(realmain);
 async fn realmain(settings: Args, executor: Arc<Executor<'_>>) -> Result<()> {
-    let buffers = create_buffers();
+    let seen = Arc::new(Mutex::new(SeenIds::new()));
 
     if settings.gen_secret {
         let secret_key = crypto_box::SecretKey::generate(&mut OsRng);
@@ -159,12 +137,12 @@ async fn realmain(settings: Args, executor: Arc<Executor<'_>>) -> Result<()> {
 
     let registry = p2p.protocol_registry();
 
-    let buffers_cloned = buffers.clone();
+    let seen_c = seen.clone();
     registry
         .register(net::SESSION_ALL, move |channel, p2p| {
             let sender = p2p_send_channel.clone();
-            let buffers_cloned = buffers_cloned.clone();
-            async move { ProtocolPrivmsg::init(channel, sender, p2p, buffers_cloned).await }
+            let seen = seen_c.clone();
+            async move { ProtocolPrivmsg::init(channel, sender, p2p, seen).await }
         })
         .await;
 
@@ -173,15 +151,7 @@ async fn realmain(settings: Args, executor: Arc<Executor<'_>>) -> Result<()> {
     let executor_cloned = executor.clone();
     executor_cloned.spawn(p2p.clone().run(executor.clone())).detach();
 
-    //
-    // Sync tasks
-    //
-    executor.spawn(resend_unread_msgs(p2p.clone(), buffers.clone())).detach();
-    executor.spawn(send_last_term(p2p.clone(), buffers.clone())).detach();
-
-    //
     // RPC interface
-    //
     let rpc_listen_addr = settings.rpc_listen.clone();
     let rpc_interface =
         Arc::new(JsonRpcInterface { addr: rpc_listen_addr.clone(), p2p: p2p.clone() });
@@ -193,7 +163,7 @@ async fn realmain(settings: Args, executor: Arc<Executor<'_>>) -> Result<()> {
 
     let ircd = Ircd::new();
 
-    ircd.start(&settings, buffers, p2p, p2p_recv_channel, executor.clone()).await?;
+    ircd.start(&settings, seen, p2p, p2p_recv_channel, executor.clone()).await?;
 
     // Run once receive exit signal
     let (signal, shutdown) = async_channel::bounded::<()>(1);

+ 12 - 193
bin/ircd/src/protocol_privmsg.rs

@@ -1,11 +1,7 @@
-use std::cmp::Ordering;
-
 use async_executor::Executor;
-use async_std::sync::Arc;
+use async_std::sync::{Arc, Mutex};
 use async_trait::async_trait;
-use chrono::Utc;
 use log::debug;
-use rand::{rngs::OsRng, RngCore};
 
 use darkfi::{
     net,
@@ -13,37 +9,7 @@ use darkfi::{
     Result,
 };
 
-use crate::{buffers::Buffers, settings, Privmsg};
-
-#[derive(SerialDecodable, SerialEncodable, Clone, Debug)]
-struct Inv {
-    id: u64,
-    invs: Vec<InvObject>,
-}
-
-#[derive(SerialDecodable, SerialEncodable, Clone, Debug)]
-pub struct LastTerm {
-    pub term: u64,
-}
-
-impl Inv {
-    fn new(invs: Vec<InvObject>) -> Self {
-        let id = OsRng.next_u64();
-        Self { id, invs }
-    }
-}
-
-#[derive(SerialDecodable, SerialEncodable, Clone, Debug)]
-struct GetData {
-    invs: Vec<InvObject>,
-    term: Option<u64>,
-}
-
-impl GetData {
-    fn new(invs: Vec<InvObject>, term: Option<u64>) -> Self {
-        Self { invs, term }
-    }
-}
+use crate::{buffers::SeenIds, Privmsg};
 
 #[derive(SerialDecodable, SerialEncodable, Clone, Debug)]
 struct InvObject(String);
@@ -52,12 +18,9 @@ pub struct ProtocolPrivmsg {
     jobsman: net::ProtocolJobsManagerPtr,
     notify: async_channel::Sender<Privmsg>,
     msg_sub: net::MessageSubscription<Privmsg>,
-    inv_sub: net::MessageSubscription<Inv>,
-    getdata_sub: net::MessageSubscription<GetData>,
-    last_term_sub: net::MessageSubscription<LastTerm>,
     p2p: net::P2pPtr,
     channel: net::ChannelPtr,
-    buffers: Buffers,
+    seen: Arc<Mutex<SeenIds>>,
 }
 
 impl ProtocolPrivmsg {
@@ -65,159 +28,41 @@ impl ProtocolPrivmsg {
         channel: net::ChannelPtr,
         notify: async_channel::Sender<Privmsg>,
         p2p: net::P2pPtr,
-        buffers: Buffers,
+        seen: Arc<Mutex<SeenIds>>,
     ) -> net::ProtocolBasePtr {
         let message_subsytem = channel.get_message_subsystem();
         message_subsytem.add_dispatch::<Privmsg>().await;
-        message_subsytem.add_dispatch::<Inv>().await;
-        message_subsytem.add_dispatch::<GetData>().await;
-        message_subsytem.add_dispatch::<LastTerm>().await;
 
         let msg_sub =
             channel.clone().subscribe_msg::<Privmsg>().await.expect("Missing Privmsg dispatcher!");
-
-        let inv_sub = channel.subscribe_msg::<Inv>().await.expect("Missing Inv dispatcher!");
-
-        let getdata_sub =
-            channel.clone().subscribe_msg::<GetData>().await.expect("Missing GetData dispatcher!");
-
-        let last_term_sub = channel
-            .clone()
-            .subscribe_msg::<LastTerm>()
-            .await
-            .expect("Missing LastTerm dispatcher!");
-
         Arc::new(Self {
             notify,
             msg_sub,
-            inv_sub,
-            getdata_sub,
-            last_term_sub,
             jobsman: net::ProtocolJobsManager::new("ProtocolPrivmsg", channel.clone()),
             p2p,
             channel,
-            buffers,
+            seen,
         })
     }
 
-    async fn handle_receive_inv(self: Arc<Self>) -> Result<()> {
-        debug!(target: "ircd", "ProtocolPrivmsg::handle_receive_inv() [START]");
-        let exclude_list = vec![self.channel.address()];
-        loop {
-            let inv = self.inv_sub.receive().await?;
-            let inv = (*inv).to_owned();
-
-            if !self.buffers.seen_ids.push(inv.id).await {
-                continue
-            }
-
-            let mut inv_requested = vec![];
-            for inv_object in inv.invs.iter() {
-                if !self.buffers.unread_msgs.inc_read_confirms(&inv_object.0).await {
-                    inv_requested.push(inv_object.clone());
-                }
-            }
-
-            if !inv_requested.is_empty() {
-                self.channel.send(GetData::new(inv_requested, None)).await?;
-            }
-
-            self.update_unread_msgs().await?;
-
-            self.p2p.broadcast_with_exclude(inv, &exclude_list).await?;
-        }
-    }
-
     async fn handle_receive_msg(self: Arc<Self>) -> Result<()> {
         debug!(target: "ircd", "ProtocolPrivmsg::handle_receive_msg() [START]");
         let exclude_list = vec![self.channel.address()];
         loop {
             let msg = self.msg_sub.receive().await?;
-            let mut msg = (*msg).to_owned();
-
-            if !self.buffers.seen_ids.push(msg.id).await {
-                continue
-            }
-
-            if msg.read_confirms >= settings::MAX_CONFIRM {
-                self.add_to_msgs(&msg).await?;
-            } else {
-                msg.read_confirms += 1;
-                let hash = self.add_to_unread_msgs(&msg).await;
-                self.p2p.broadcast(Inv::new(vec![InvObject(hash)])).await?;
-            }
-
-            self.update_unread_msgs().await?;
-
-            self.p2p.broadcast_with_exclude(msg, &exclude_list).await?;
-        }
-    }
-
-    async fn handle_receive_last_term(self: Arc<Self>) -> Result<()> {
-        debug!(target: "ircd", "ProtocolPrivmsg::handle_receive_last_term() [START]");
-        loop {
-            let last_term = self.last_term_sub.receive().await?;
-            let last_term = last_term.term;
-
-            self.update_unread_msgs().await?;
+            let msg = (*msg).to_owned();
 
-            match self.buffers.privmsgs.last_term().await.cmp(&last_term) {
-                Ordering::Greater => {
-                    for msg in self.buffers.privmsgs.fetch_msgs(last_term).await {
-                        self.channel.send(msg).await?;
-                    }
-                }
-                Ordering::Less => {
-                    self.channel.send(GetData::new(vec![], Some(last_term))).await?;
+            {
+                let ids = &mut self.seen.lock().await;
+                if !ids.push(msg.id) {
+                    continue
                 }
-                Ordering::Equal => continue,
             }
-        }
-    }
 
-    async fn handle_receive_getdata(self: Arc<Self>) -> Result<()> {
-        debug!(target: "ircd", "ProtocolPrivmsg::handle_receive_getdata() [START]");
-        loop {
-            let getdata = self.getdata_sub.receive().await?;
-            let getdata = (*getdata).to_owned();
+            self.notify.send(msg.clone()).await?;
 
-            for inv in getdata.invs {
-                if let Some(msg) = self.buffers.unread_msgs.get(&inv.0).await {
-                    self.channel.send(msg.clone()).await?;
-                }
-            }
-
-            if let Some(term) = getdata.term {
-                for msg in self.buffers.privmsgs.fetch_msgs(term).await {
-                    self.channel.send(msg).await?;
-                }
-            }
-        }
-    }
-
-    async fn add_to_unread_msgs(&self, msg: &Privmsg) -> String {
-        self.buffers.unread_msgs.insert(msg).await
-    }
-
-    async fn update_unread_msgs(&self) -> Result<()> {
-        for (hash, msg) in self.buffers.unread_msgs.load().await {
-            if msg.timestamp + settings::UNREAD_MSG_EXPIRE_TIME < Utc::now().timestamp() {
-                self.buffers.unread_msgs.remove(&hash).await;
-                continue
-            }
-            if msg.read_confirms >= settings::MAX_CONFIRM {
-                if let Some(msg) = self.buffers.unread_msgs.remove(&hash).await {
-                    self.add_to_msgs(&msg).await?;
-                }
-            }
+            self.p2p.broadcast_with_exclude(msg, &exclude_list).await?;
         }
-        Ok(())
-    }
-
-    async fn add_to_msgs(&self, msg: &Privmsg) -> Result<()> {
-        self.buffers.privmsgs.push(msg).await;
-        self.notify.send(msg.clone()).await?;
-        Ok(())
     }
 }
 
@@ -227,17 +72,9 @@ impl net::ProtocolBase for ProtocolPrivmsg {
     /// protocol task manager, then queues the reply. Sends out a ping and
     /// waits for pong reply. Waits for ping and replies with a pong.
     async fn start(self: Arc<Self>, executor: Arc<Executor<'_>>) -> Result<()> {
-        // once a channel get started
-        for m in self.buffers.privmsgs.load().await {
-            self.channel.send(m).await?;
-        }
-
         debug!(target: "ircd", "ProtocolPrivmsg::start() [START]");
         self.jobsman.clone().start(executor.clone());
         self.jobsman.clone().spawn(self.clone().handle_receive_msg(), executor.clone()).await;
-        self.jobsman.clone().spawn(self.clone().handle_receive_inv(), executor.clone()).await;
-        self.jobsman.clone().spawn(self.clone().handle_receive_getdata(), executor.clone()).await;
-        self.jobsman.clone().spawn(self.clone().handle_receive_last_term(), executor.clone()).await;
         debug!(target: "ircd", "ProtocolPrivmsg::start() [END]");
         Ok(())
     }
@@ -252,21 +89,3 @@ impl net::Message for Privmsg {
         "privmsg"
     }
 }
-
-impl net::Message for Inv {
-    fn name() -> &'static str {
-        "inv"
-    }
-}
-
-impl net::Message for GetData {
-    fn name() -> &'static str {
-        "getdata"
-    }
-}
-
-impl net::Message for LastTerm {
-    fn name() -> &'static str {
-        "last_term"
-    }
-}