Quellcode durchsuchen

wrapping darkwiki code up & create darkwiki cli

ghassmo vor 4 Jahren
Ursprung
Commit
81e14dd213

+ 49 - 0
Cargo.lock

@@ -1321,6 +1321,55 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "darkwiki"
+version = "0.3.0"
+dependencies = [
+ "async-channel",
+ "async-std",
+ "async-trait",
+ "darkfi",
+ "futures",
+ "log",
+ "rand",
+ "serde",
+ "serde_json",
+ "simplelog",
+ "smol",
+ "structopt",
+ "url",
+]
+
+[[package]]
+name = "darkwikid"
+version = "0.3.0"
+dependencies = [
+ "async-channel",
+ "async-executor",
+ "async-std",
+ "async-trait",
+ "bs58",
+ "chrono",
+ "crypto_box",
+ "ctrlc-async",
+ "darkfi",
+ "easy-parallel",
+ "futures",
+ "fxhash",
+ "hex",
+ "log",
+ "rand",
+ "serde",
+ "serde_json",
+ "simplelog",
+ "smol",
+ "structopt",
+ "structopt-toml",
+ "thiserror",
+ "unicode-segmentation",
+ "url",
+]
+
 [[package]]
 name = "darling"
 version = "0.10.2"

+ 2 - 0
Cargo.toml

@@ -20,6 +20,8 @@ name = "darkfi"
 members = [
 	"bin/zkas",
 	#"bin/cashierd",
+	"bin/darkwiki",
+	"bin/darkwikid",
 	"bin/darkfid",
 	"bin/darkotc",
 	"bin/drk",

+ 42 - 0
bin/darkwiki/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "darkwiki"
+description = ""
+version = "0.3.0"
+edition = "2021"
+authors = ["darkfi <dev@dark.fi>"]
+license = "AGPL-3.0-only"
+homepage = "https://dark.fi"
+repository = "https://github.com/darkrenaissance/darkfi"
+keywords = []
+categories = []
+
+
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+darkfi = {path = "../../", features = ["rpc"]}
+
+
+# Async
+smol = "1.2.5"
+async-std = {version = "1.12.0", features = ["attributes"]}
+async-trait = "0.1.57"
+async-channel = "1.6.1"
+futures = "0.3.21"
+
+# Misc
+log = "0.4.17"
+simplelog = "0.12.0"
+rand = "0.8.5"
+url = "2.2.2"
+
+# Encoding and parsing
+serde = {version = "1.0.142", features = ["derive"]}
+serde_json = "1.0.83"
+structopt = "0.3.26"
+
+
+
+
+

+ 0 - 0
script/research/darkwiki/src/update_cli.rs → bin/darkwiki/src/main.rs


+ 0 - 0
script/research/darkwiki/.gitignore → bin/darkwikid/.gitignore


+ 11 - 15
script/research/darkwiki/Cargo.toml → bin/darkwikid/Cargo.toml

@@ -1,12 +1,19 @@
 [package]
-name = "darkwiki"
-version = "0.1.0"
+name = "darkwikid"
+description = ""
+version = "0.3.0"
 edition = "2021"
+authors = ["darkfi <dev@dark.fi>"]
+license = "AGPL-3.0-only"
+homepage = "https://dark.fi"
+repository = "https://github.com/darkrenaissance/darkfi"
+keywords = []
+categories = []
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-darkfi = {path = "../../../", features = ["raft"]}
+darkfi = {path = "../../", features = ["raft", "net", "rpc"]}
 
 
 # Async
@@ -34,19 +41,8 @@ serde_json = "1.0.83"
 structopt = "0.3.26"
 structopt-toml = "0.5.1"
 unicode-segmentation = "1.9.0"
-crypto_box = {version = "0.7.2", features = ["std"]}
+crypto_box = {version = "0.8.0", features = ["std"]}
 hex = "0.4.3"
 bs58 = "0.4.0"
 
 
-[workspace]
-
-[[bin]]
-name = "darkwikid"
-path = "src/main.rs"
-
-[[bin]]
-name = "darkwikiupdate"
-path = "src/update_cli.rs"
-
-

+ 4 - 1
script/research/darkwiki/darkwiki.toml → bin/darkwikid/darkwiki.toml

@@ -4,13 +4,16 @@
 ## Sets Datastore Path
 #datastore="~/.config/darkfi/darkwiki"
 
+## Sets Docs Path
+# docs="~/darkwiki"
+
 ## Raft net settings
 [net]
 ## P2P accept address
 inbound="tcp://127.0.0.1:13001" 
 
 ## Connection slots
-#outbound_connections=5
+outbound_connections=2
 
 ## P2P external address
 external_addr="tls://127.0.0.1:13001"

+ 0 - 2
script/research/darkwiki/src/error.rs → bin/darkwikid/src/error.rs

@@ -1,7 +1,5 @@
 #[derive(thiserror::Error, Debug)]
 pub enum DarkWikiError {
-    #[error("Add Operation failed")]
-    AddOperationFailed,
     #[error("Encryption error: `{0}`")]
     EncryptionError(String),
     #[error("Json serialization error: `{0}`")]

+ 0 - 0
script/research/darkwiki/src/jsonrpc.rs → bin/darkwikid/src/jsonrpc.rs


+ 322 - 0
bin/darkwikid/src/main.rs

@@ -0,0 +1,322 @@
+use async_std::sync::{Arc, Mutex};
+use std::{
+    fs::{create_dir_all, read_dir},
+    mem::discriminant,
+    path::PathBuf,
+};
+
+use async_executor::Executor;
+use futures::{select, FutureExt};
+use fxhash::FxHashMap;
+use log::{error, info, warn};
+use serde::Deserialize;
+use smol::future;
+use structopt::StructOpt;
+use structopt_toml::StructOptToml;
+use unicode_segmentation::UnicodeSegmentation;
+use url::Url;
+
+use darkfi::{
+    async_daemonize,
+    net::{self, settings::SettingsOpt},
+    raft::{NetMsg, ProtocolRaft, Raft, RaftSettings},
+    rpc::server::listen_and_serve,
+    util::{
+        cli::{get_log_config, get_log_level, spawn_config},
+        expand_path,
+        file::{load_file, load_json_file, save_file, save_json_file},
+        path::get_config_path,
+    },
+    Error, Result,
+};
+
+mod error;
+mod jsonrpc;
+mod patch;
+
+use error::DarkWikiResult;
+use jsonrpc::JsonRpcInterface;
+use patch::{OpMethod, Patch};
+
+pub const CONFIG_FILE: &str = "darkwiki.toml";
+pub const CONFIG_FILE_CONTENTS: &str = include_str!("../darkwiki.toml");
+
+/// darkwikid cli
+#[derive(Clone, Debug, Deserialize, StructOpt, StructOptToml)]
+#[serde(default)]
+#[structopt(name = "darkwikid")]
+pub struct Args {
+    /// Sets a custom config file
+    #[structopt(long)]
+    pub config: Option<String>,
+    /// Sets Datastore Path
+    #[structopt(long, default_value = "~/.config/darkfi/darkwiki")]
+    pub datastore: String,
+    /// Sets Docs Path
+    #[structopt(long, default_value = "~/darkwiki")]
+    pub docs: String,
+    /// Sets Author Name for Patch
+    #[structopt(long, default_value = "NONE")]
+    pub author: String,
+    /// JSON-RPC listen URL
+    #[structopt(long = "rpc", default_value = "tcp://127.0.0.1:13055")]
+    pub rpc_listen: Url,
+    #[structopt(flatten)]
+    pub net: SettingsOpt,
+    /// Increase verbosity
+    #[structopt(short, parse(from_occurrences))]
+    pub verbose: u8,
+}
+
+pub struct DarkWikiSettings {
+    author: String,
+    docs_path: PathBuf,
+    datastore_path: PathBuf,
+}
+
+fn str_to_chars(s: &str) -> Vec<&str> {
+    s.graphemes(true).collect::<Vec<&str>>()
+}
+
+fn lcs(a: &str, b: &str) -> Vec<OpMethod> {
+    let a: Vec<_> = str_to_chars(a);
+    let b: Vec<_> = str_to_chars(b);
+    let (na, nb) = (a.len(), b.len());
+
+    let mut lengths = vec![vec![0; nb + 1]; na + 1];
+
+    for (i, ci) in a.iter().enumerate() {
+        for (j, cj) in b.iter().enumerate() {
+            lengths[i + 1][j + 1] =
+                if ci == cj { lengths[i][j] + 1 } else { lengths[i][j + 1].max(lengths[i + 1][j]) }
+        }
+    }
+
+    let mut result = Vec::new();
+    let (mut i, mut j) = (na, nb);
+    while i > 0 && j > 0 {
+        if a[i - 1] == b[j - 1] {
+            result.push(OpMethod::Retain((1) as _));
+            i -= 1;
+            j -= 1;
+        } else if lengths[i - 1][j] > lengths[i][j - 1] {
+            result.push(OpMethod::Delete((1) as _));
+            i -= 1;
+        } else {
+            result.push(OpMethod::Insert(b[j - 1].to_string()));
+            j -= 1;
+        }
+    }
+
+    result.reverse();
+    result
+}
+
+fn on_receive_patch(received_patch: &Patch, settings: &DarkWikiSettings) -> DarkWikiResult<()> {
+    let sync_id_path = settings.datastore_path.join("sync").join(&received_patch.id());
+    let local_id_path = settings.datastore_path.join("local").join(&received_patch.id());
+
+    if let Ok(mut sync_patch) = load_json_file::<Patch>(&sync_id_path) {
+        if let Ok(local_edit) = load_file(&local_id_path) {
+            let local_edit = local_edit.trim();
+
+            if local_edit == sync_patch.to_string() {
+                sync_patch.set_ops(received_patch.ops());
+            } else {
+                sync_patch.extend_ops(received_patch.ops());
+            }
+        }
+
+        save_json_file::<Patch>(&sync_id_path, &sync_patch)?;
+    } else if !received_patch.base_empty() {
+        save_json_file::<Patch>(&sync_id_path, received_patch)?;
+    }
+
+    Ok(())
+}
+
+fn on_receive_update(settings: &DarkWikiSettings) -> DarkWikiResult<Vec<Patch>> {
+    let mut patches: Vec<Patch> = vec![];
+
+    let local_path = settings.datastore_path.join("local");
+    let sync_path = settings.datastore_path.join("sync");
+    let docs_path = settings.docs_path.clone();
+
+    // save and compare docs in darkwiki and local dirs
+    // then merged with sync patches if any received
+    let docs = read_dir(&docs_path).map_err(Error::from)?;
+    for doc in docs {
+        let doc_id = doc.as_ref().unwrap().file_name();
+        let doc_path = docs_path.join(&doc_id);
+
+        // load doc content
+        let edit = load_file(&doc_path).map_err(Error::from)?;
+        let edit = edit.trim();
+
+        // create new patch
+        let mut new_patch = Patch::new(doc_id.to_str().unwrap(), &settings.author);
+
+        // check for any changes found with local doc and darkwiki doc
+        if let Ok(local_edit) = load_file(&local_path.join(&doc_id)) {
+            let local_edit = local_edit.trim();
+
+            // check the differences with LCS algorithm
+            let lcs_ops = lcs(local_edit, edit);
+
+            let retains_len = lcs_ops
+                .iter()
+                .filter(|&o| discriminant(&OpMethod::Retain(0)) == discriminant(o))
+                .count();
+
+            // if all the ops in lcs_ops are Reatin then no changes found
+            if retains_len == lcs_ops.len() {
+                continue
+            }
+
+            // add the change ops to the new patch
+            for op in lcs_ops {
+                new_patch.add_op(&op);
+            }
+
+            // check if the same doc has received patch from the network
+            if let Ok(sync_patch) = load_json_file::<Patch>(&sync_path.join(&doc_id)) {
+                if sync_patch.to_string() != local_edit {
+                    let sync_patch_t = new_patch.transform(&sync_patch);
+                    new_patch = new_patch.merge(&sync_patch_t);
+                    save_file(&doc_path, &new_patch.to_string())?;
+                }
+            }
+        } else {
+            new_patch.set_base(edit);
+        };
+
+        save_file(&local_path.join(&doc_id), &new_patch.to_string())?;
+        save_json_file(&sync_path.join(doc_id), &new_patch)?;
+        patches.push(new_patch);
+    }
+
+    // check if a new patch received
+    // and save the new changes in both local and darkwiki dirs
+    let sync_files = read_dir(&sync_path).map_err(Error::from)?;
+    for file in sync_files {
+        let file_id = file.as_ref().unwrap().file_name();
+        let file_path = sync_path.join(&file_id);
+        let sync_patch: Patch = load_json_file(&file_path)?;
+
+        if let Ok(local_edit) = load_file(&local_path.join(&file_id)) {
+            if local_edit.trim() == sync_patch.to_string() {
+                continue
+            }
+        }
+
+        save_file(&docs_path.join(&file_id), &sync_patch.to_string())?;
+        save_file(&local_path.join(file_id), &sync_patch.to_string())?;
+    }
+
+    Ok(patches)
+}
+
+async fn start(
+    update_notifier_rv: async_channel::Receiver<()>,
+    raft_sender: async_channel::Sender<Patch>,
+    raft_receiver: async_channel::Receiver<Patch>,
+    settings: DarkWikiSettings,
+) -> DarkWikiResult<()> {
+    loop {
+        select! {
+            _ = update_notifier_rv.recv().fuse() => {
+                let patches = on_receive_update(&settings)?;
+                for patch in patches {
+                    info!("Send a patch to Raft {:?}", patch);
+                    raft_sender.send(patch).await.map_err(Error::from)?;
+                }
+            }
+            patch = raft_receiver.recv().fuse() => {
+                let patch = patch.map_err(Error::from)?;
+                info!("Receive new patch from Raft {:?}", patch);
+                on_receive_patch(&patch, &settings)?;
+            }
+
+        }
+    }
+}
+
+async_daemonize!(realmain);
+async fn realmain(settings: Args, executor: Arc<Executor<'_>>) -> Result<()> {
+    let datastore_path = expand_path(&settings.datastore)?;
+    let docs_path = expand_path(&settings.docs)?;
+
+    create_dir_all(docs_path.clone())?;
+    create_dir_all(datastore_path.join("local"))?;
+    create_dir_all(datastore_path.join("sync"))?;
+
+    let (update_notifier_sx, update_notifier_rv) = async_channel::unbounded::<()>();
+
+    //
+    // RPC
+    //
+    let rpc_interface = Arc::new(JsonRpcInterface::new(update_notifier_sx));
+    executor.spawn(listen_and_serve(settings.rpc_listen.clone(), rpc_interface)).detach();
+
+    //
+    // Raft
+    //
+    let net_settings = settings.net;
+    let seen_net_msgs = Arc::new(Mutex::new(FxHashMap::default()));
+
+    let datastore_raft = datastore_path.join("darkwiki.db");
+    let raft_settings = RaftSettings { datastore_path: datastore_raft, ..RaftSettings::default() };
+
+    let mut raft = Raft::<Patch>::new(raft_settings, seen_net_msgs.clone())?;
+
+    //
+    // P2p setup
+    //
+    let (p2p_send_channel, p2p_recv_channel) = async_channel::unbounded::<NetMsg>();
+
+    let p2p = net::P2p::new(net_settings.into()).await;
+    let p2p = p2p.clone();
+
+    let registry = p2p.protocol_registry();
+
+    let raft_node_id = raft.id();
+    registry
+        .register(net::SESSION_ALL, move |channel, p2p| {
+            let raft_node_id = raft_node_id.clone();
+            let sender = p2p_send_channel.clone();
+            let seen_net_msgs_cloned = seen_net_msgs.clone();
+            async move {
+                ProtocolRaft::init(raft_node_id, channel, sender, p2p, seen_net_msgs_cloned).await
+            }
+        })
+        .await;
+
+    p2p.clone().start(executor.clone()).await?;
+
+    executor.spawn(p2p.clone().run(executor.clone())).detach();
+
+    //
+    // Darkwiki start
+    //
+    let darkwiki_settings = DarkWikiSettings { author: settings.author, datastore_path, docs_path };
+    executor
+        .spawn(start(update_notifier_rv, raft.sender(), raft.receiver(), darkwiki_settings))
+        .detach();
+
+    //
+    // Waiting Exit signal
+    //
+    let (signal, shutdown) = async_channel::bounded::<()>(1);
+    ctrlc_async::set_async_handler(async move {
+        warn!(target: "darkwiki", "Catch exit signal");
+        // cleaning up tasks running in the background
+        if let Err(e) = signal.send(()).await {
+            error!("Error on sending exit signal: {}", e);
+        }
+    })
+    .unwrap();
+
+    raft.run(p2p.clone(), p2p_recv_channel.clone(), executor.clone(), shutdown.clone()).await?;
+
+    Ok(())
+}

+ 535 - 0
bin/darkwikid/src/patch.rs

@@ -0,0 +1,535 @@
+use std::{cmp::Ordering, io};
+
+use serde::{Deserialize, Serialize};
+
+use darkfi::util::serial::{Decodable, Encodable, SerialDecodable, SerialEncodable, VarInt};
+
+use crate::str_to_chars;
+
+#[derive(PartialEq, Eq, Serialize, Deserialize, Clone, Debug)]
+pub enum OpMethod {
+    Delete(u64),
+    Insert(String),
+    Retain(u64),
+}
+
+#[derive(PartialEq, Eq, Serialize, Deserialize, Clone, Debug)]
+pub struct OpMethods(pub Vec<OpMethod>);
+
+#[derive(PartialEq, Eq, SerialEncodable, SerialDecodable, Serialize, Deserialize, Clone, Debug)]
+pub struct Patch {
+    author: String,
+    id: String,
+    base: String,
+    ops: OpMethods,
+}
+
+impl std::string::ToString for Patch {
+    fn to_string(&self) -> String {
+        let mut st = vec![];
+        let mut index: usize = 0;
+
+        st.extend(str_to_chars(&self.base));
+        for op in self.ops.0.iter() {
+            match op {
+                OpMethod::Retain(n) => {
+                    index += *n as usize;
+                }
+                OpMethod::Delete(n) => {
+                    if (index + (*n as usize)) > st.len() {
+                        if index < st.len() {
+                            st.drain(index..st.len());
+                        }
+                    } else {
+                        st.drain(index..(index + *n as usize));
+                    }
+                }
+                OpMethod::Insert(insert) => {
+                    let chars = str_to_chars(insert);
+                    for c in chars {
+                        if index > st.len() {
+                            st.push(c);
+                        } else {
+                            st.insert(index, c);
+                            index += 1;
+                        }
+                    }
+                }
+            }
+        }
+
+        st.join("")
+    }
+}
+
+impl Patch {
+    pub fn new(id: &str, author: &str) -> Self {
+        Self {
+            id: id.to_string(),
+            ops: OpMethods(vec![]),
+            base: String::new(),
+            author: author.to_string(),
+        }
+    }
+
+    pub fn add_op(&mut self, method: &OpMethod) {
+        match method {
+            OpMethod::Delete(n) => {
+                if *n == 0 {
+                    return
+                }
+
+                if let Some(OpMethod::Delete(i)) = self.ops.0.last_mut() {
+                    *i += n;
+                } else {
+                    self.ops.0.push(method.to_owned());
+                }
+            }
+            OpMethod::Insert(insert) => {
+                if insert.is_empty() {
+                    return
+                }
+
+                if let Some(OpMethod::Insert(s)) = self.ops.0.last_mut() {
+                    *s += insert;
+                } else {
+                    self.ops.0.push(OpMethod::Insert(insert.to_owned()));
+                }
+            }
+            OpMethod::Retain(n) => {
+                if *n == 0 {
+                    return
+                }
+
+                if let Some(OpMethod::Retain(i)) = self.ops.0.last_mut() {
+                    *i += n;
+                } else {
+                    self.ops.0.push(method.to_owned());
+                }
+            }
+        }
+    }
+
+    fn insert(&mut self, st: &str) {
+        self.add_op(&OpMethod::Insert(st.into()));
+    }
+
+    fn retain(&mut self, n: u64) {
+        self.add_op(&OpMethod::Retain(n));
+    }
+
+    fn delete(&mut self, n: u64) {
+        self.add_op(&OpMethod::Delete(n));
+    }
+
+    pub fn set_base(&mut self, base: &str) {
+        self.base = base.to_owned();
+    }
+
+    pub fn set_ops(&mut self, ops: OpMethods) {
+        self.ops = ops;
+    }
+
+    pub fn extend_ops(&mut self, ops: OpMethods) {
+        self.ops.0.extend(ops.0);
+    }
+
+    pub fn base_empty(&self) -> bool {
+        self.base.is_empty()
+    }
+
+    pub fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    pub fn ops(&self) -> OpMethods {
+        self.ops.clone()
+    }
+
+    //
+    // these two functions are imported from this library
+    // https://github.com/spebern/operational-transform-rs
+    // with some major modification
+    //
+    // TODO need more work to get better performance with iterators
+    pub fn transform(&self, other: &Self) -> Self {
+        let mut new_patch = Self::new(&self.id, &self.author);
+        new_patch.set_base(&self.base);
+
+        let mut ops1 = self.ops.0.iter().cloned();
+        let mut ops2 = other.ops.0.iter().cloned();
+
+        let mut op1 = ops1.next();
+        let mut op2 = ops2.next();
+        loop {
+            if op2.is_none() {
+                break
+            }
+
+            if op1.is_none() {
+                new_patch.add_op(op2.as_ref().unwrap());
+                op2 = ops2.next();
+                continue
+            }
+
+            match (op1.as_ref().unwrap(), op2.as_ref().unwrap()) {
+                (OpMethod::Insert(s), _) => {
+                    new_patch.retain(str_to_chars(s).len() as _);
+                    op1 = ops1.next();
+                }
+                (_, OpMethod::Insert(s)) => {
+                    new_patch.insert(s);
+                    op2 = ops2.next();
+                }
+                (OpMethod::Retain(i), OpMethod::Retain(j)) => match i.cmp(j) {
+                    Ordering::Less => {
+                        new_patch.retain(*i);
+                        op2 = Some(OpMethod::Retain(j - *i));
+                        op1 = ops1.next();
+                    }
+                    Ordering::Greater => {
+                        new_patch.retain(*j);
+                        op1 = Some(OpMethod::Retain(i - j));
+                        op2 = ops2.next();
+                    }
+                    Ordering::Equal => {
+                        new_patch.retain(*i);
+                        op1 = ops1.next();
+                        op2 = ops2.next();
+                    }
+                },
+                (OpMethod::Delete(i), OpMethod::Delete(j)) => match i.cmp(j) {
+                    Ordering::Less => {
+                        op2 = Some(OpMethod::Delete(j - *i));
+                        op1 = ops1.next();
+                    }
+                    Ordering::Greater => {
+                        op1 = Some(OpMethod::Delete(i - j));
+                        op2 = ops2.next();
+                    }
+                    Ordering::Equal => {
+                        op1 = ops1.next();
+                        op2 = ops2.next();
+                    }
+                },
+                (OpMethod::Delete(i), OpMethod::Retain(j)) => match i.cmp(j) {
+                    Ordering::Less => {
+                        op2 = Some(OpMethod::Retain(j - *i));
+                        op1 = ops1.next();
+                    }
+                    Ordering::Greater => {
+                        op1 = Some(OpMethod::Delete(i - j));
+                        op2 = ops2.next();
+                    }
+                    Ordering::Equal => {
+                        op1 = ops1.next();
+                        op2 = ops2.next();
+                    }
+                },
+                (OpMethod::Retain(i), OpMethod::Delete(j)) => match i.cmp(j) {
+                    Ordering::Less => {
+                        new_patch.delete(*i);
+                        op2 = Some(OpMethod::Delete(j - i));
+                        op1 = ops1.next();
+                    }
+                    Ordering::Greater => {
+                        new_patch.delete(*j);
+                        op1 = Some(OpMethod::Retain(i - j));
+                        op2 = ops2.next();
+                    }
+                    Ordering::Equal => {
+                        new_patch.delete(*i);
+                        op1 = ops1.next();
+                        op2 = ops2.next();
+                    }
+                },
+            }
+        }
+
+        new_patch
+    }
+
+    // TODO need more work to get better performance with iterators
+    pub fn merge(&mut self, other: &Self) -> Self {
+        let ops1 = self.ops.0.clone();
+        let mut ops1 = ops1.iter().cloned();
+        let mut ops2 = other.ops.0.iter().cloned();
+
+        let mut new_patch = Self::new(&self.id, &self.author);
+        new_patch.set_base(&self.base);
+
+        let mut op1 = ops1.next();
+        let mut op2 = ops2.next();
+
+        loop {
+            if op2.is_none() {
+                break
+            }
+
+            if op1.is_none() {
+                new_patch.add_op(op2.as_ref().unwrap());
+                op2 = ops2.next();
+                continue
+            }
+
+            match (op1.as_ref().unwrap(), op2.as_ref().unwrap()) {
+                (OpMethod::Delete(i), _) => {
+                    new_patch.delete(*i);
+                    op1 = ops1.next();
+                }
+                (_, OpMethod::Insert(s)) => {
+                    new_patch.insert(s);
+                    op2 = ops2.next();
+                }
+                (OpMethod::Retain(i), OpMethod::Retain(j)) => match i.cmp(j) {
+                    Ordering::Less => {
+                        new_patch.retain(*i);
+                        op2 = Some(OpMethod::Retain(*j - i));
+                        op1 = ops1.next();
+                    }
+                    Ordering::Greater => {
+                        new_patch.retain(*j);
+                        op1 = Some(OpMethod::Retain(i - *j));
+                        op2 = ops2.next();
+                    }
+                    Ordering::Equal => {
+                        new_patch.retain(*i);
+                        op1 = ops1.next();
+                        op2 = ops2.next();
+                    }
+                },
+                (OpMethod::Insert(s), OpMethod::Delete(j)) => {
+                    let chars = str_to_chars(s);
+                    let chars_len = chars.len() as u64;
+                    match chars_len.cmp(j) {
+                        Ordering::Less => {
+                            op1 = ops1.next();
+                            op2 = Some(OpMethod::Delete(j - chars_len));
+                        }
+                        Ordering::Greater => {
+                            let st = chars.into_iter().skip(*j as usize).collect();
+                            op1 = Some(OpMethod::Insert(st));
+                            op2 = ops2.next();
+                        }
+                        Ordering::Equal => {
+                            op1 = ops1.next();
+                            op2 = ops2.next();
+                        }
+                    }
+                }
+                (OpMethod::Insert(s), OpMethod::Retain(j)) => {
+                    let chars = str_to_chars(s);
+                    let chars_len = chars.len() as u64;
+                    match chars_len.cmp(j) {
+                        Ordering::Less => {
+                            new_patch.insert(s);
+                            op1 = ops1.next();
+                            op2 = Some(OpMethod::Retain(*j - chars_len));
+                        }
+                        Ordering::Greater => {
+                            let st = chars.into_iter().take(*j as usize).collect::<String>();
+                            new_patch.insert(&st);
+                            op1 = Some(OpMethod::Insert(st));
+                            op2 = ops2.next();
+                        }
+                        Ordering::Equal => {
+                            new_patch.insert(s);
+                            op1 = ops1.next();
+                            op2 = ops2.next();
+                        }
+                    }
+                }
+                (OpMethod::Retain(i), OpMethod::Delete(j)) => match i.cmp(j) {
+                    Ordering::Less => {
+                        new_patch.delete(*i);
+                        op2 = Some(OpMethod::Delete(*j - *i));
+                        op1 = ops1.next();
+                    }
+                    Ordering::Greater => {
+                        new_patch.delete(*j);
+                        op1 = Some(OpMethod::Retain(*i - *j));
+                        op2 = ops2.next();
+                    }
+                    Ordering::Equal => {
+                        new_patch.delete(*j);
+                        op1 = ops1.next();
+                        op2 = ops2.next();
+                    }
+                },
+            };
+        }
+
+        new_patch
+    }
+}
+
+impl Decodable for OpMethod {
+    fn decode<D: io::Read>(mut d: D) -> darkfi::Result<Self> {
+        let com: u8 = Decodable::decode(&mut d)?;
+        match com {
+            0 => {
+                let i: u64 = Decodable::decode(&mut d)?;
+                Ok(Self::Delete(i))
+            }
+            1 => {
+                let t: String = Decodable::decode(d)?;
+                Ok(Self::Insert(t))
+            }
+            2 => {
+                let i: u64 = Decodable::decode(&mut d)?;
+                Ok(Self::Retain(i))
+            }
+            _ => Err(darkfi::Error::ParseFailed("Parse OpMethod failed")),
+        }
+    }
+}
+
+impl Encodable for OpMethod {
+    fn encode<S: io::Write>(&self, mut s: S) -> darkfi::Result<usize> {
+        let len: usize = match self {
+            Self::Delete(i) => (0_u8).encode(&mut s)? + i.encode(&mut s)?,
+            Self::Insert(t) => (1_u8).encode(&mut s)? + t.encode(&mut s)?,
+            Self::Retain(i) => (2_u8).encode(&mut s)? + i.encode(&mut s)?,
+        };
+        Ok(len)
+    }
+}
+
+impl Encodable for OpMethods {
+    fn encode<S: io::Write>(&self, mut s: S) -> darkfi::Result<usize> {
+        let mut len = 0;
+        len += VarInt(self.0.len() as u64).encode(&mut s)?;
+        for c in self.0.iter() {
+            len += c.encode(&mut s)?;
+        }
+        Ok(len)
+    }
+}
+
+impl Decodable for OpMethods {
+    fn decode<D: io::Read>(mut d: D) -> darkfi::Result<Self> {
+        let len = VarInt::decode(&mut d)?.0;
+        let mut ret = Vec::with_capacity(len as usize);
+        for _ in 0..len {
+            ret.push(Decodable::decode(&mut d)?);
+        }
+        Ok(Self(ret))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use darkfi::util::{
+        gen_id,
+        serial::{deserialize, serialize},
+    };
+
+    #[test]
+    fn test_to_string() {
+        let mut patch = Patch::new(&gen_id(30), "");
+        patch.set_base("text example\n hello");
+        patch.retain(14);
+        patch.delete(5);
+        patch.insert("hey");
+
+        assert_eq!(patch.to_string(), "text example\n hey");
+    }
+
+    #[test]
+    fn test_merge() {
+        let mut patch_init = Patch::new(&gen_id(30), "");
+        let base = "text example\n hello";
+        patch_init.set_base(base);
+
+        let mut patch1 = patch_init.clone();
+        patch1.retain(14);
+        patch1.delete(5);
+        patch1.insert("hey");
+
+        let mut patch2 = patch_init.clone();
+        patch2.retain(14);
+        patch2.delete(5);
+        patch2.insert("test");
+
+        patch1.merge(&patch2);
+
+        let patch3 = patch1.merge(&patch2);
+
+        assert_eq!(patch3.to_string(), "text example\n test");
+
+        let mut patch1 = patch_init.clone();
+        patch1.retain(5);
+        patch1.delete(7);
+        patch1.insert("ex");
+        patch1.retain(7);
+
+        let mut patch2 = patch_init.clone();
+        patch2.delete(4);
+        patch2.insert("new");
+        patch2.retain(13);
+
+        let patch3 = patch1.merge(&patch2);
+
+        assert_eq!(patch3.to_string(), "new ex\n hello");
+    }
+
+    #[test]
+    fn test_transform() {
+        let mut patch_init = Patch::new(&gen_id(30), "");
+        let base = "text example\n hello";
+        patch_init.set_base(base);
+
+        let mut patch1 = patch_init.clone();
+        patch1.retain(14);
+        patch1.delete(5);
+        patch1.insert("hey");
+
+        let mut patch2 = patch_init.clone();
+        patch2.retain(14);
+        patch2.delete(5);
+        patch2.insert("test");
+
+        let patch3 = patch1.transform(&patch2);
+        let patch4 = patch1.merge(&patch3);
+
+        assert_eq!(patch4.to_string(), "text example\n heytest");
+
+        let mut patch1 = patch_init.clone();
+        patch1.retain(5);
+        patch1.delete(7);
+        patch1.insert("ex");
+        patch1.retain(7);
+
+        let mut patch2 = patch_init.clone();
+        patch2.delete(4);
+        patch2.insert("new");
+        patch2.retain(13);
+
+        let patch3 = patch1.transform(&patch2);
+        let patch4 = patch1.merge(&patch3);
+
+        assert_eq!(patch4.to_string(), "new ex\n hello");
+    }
+
+    #[test]
+    fn test_serialize() {
+        // serialize & deserialize OpMethod
+        let op_method = OpMethod::Delete(3);
+
+        let op_method_ser = serialize(&op_method);
+        let op_method_deser = deserialize(&op_method_ser).unwrap();
+
+        assert_eq!(op_method, op_method_deser);
+
+        // serialize & deserialize Patch
+        let mut patch = Patch::new(&gen_id(30), "");
+        patch.insert("hello");
+        patch.delete(2);
+
+        let patch_ser = serialize(&patch);
+        let patch_deser = deserialize(&patch_ser).unwrap();
+
+        assert_eq!(patch, patch_deser);
+    }
+}

+ 0 - 205
script/research/darkwiki/src/main.rs

@@ -1,205 +0,0 @@
-use async_std::sync::{Arc, Mutex};
-use std::{
-    fs::{create_dir_all, read_dir},
-    path::{Path, PathBuf},
-};
-
-use async_executor::Executor;
-use futures::{select, FutureExt};
-use fxhash::FxHashMap;
-use log::{error, warn};
-use serde::Deserialize;
-use smol::future;
-use structopt::StructOpt;
-use structopt_toml::StructOptToml;
-use url::Url;
-
-use darkfi::{
-    async_daemonize,
-    net::{self, settings::SettingsOpt},
-    raft::{NetMsg, ProtocolRaft, Raft, RaftSettings},
-    rpc::server::listen_and_serve,
-    util::{
-        cli::{get_log_config, get_log_level, spawn_config},
-        expand_path,
-        file::{load_file, load_json_file, save_file, save_json_file},
-        gen_id,
-        path::get_config_path,
-    },
-    Error, Result,
-};
-
-mod error;
-mod jsonrpc;
-mod sequence;
-
-use error::DarkWikiResult;
-use jsonrpc::JsonRpcInterface;
-use sequence::{Operation, Sequence};
-
-pub const CONFIG_FILE: &str = "darkwiki.toml";
-pub const CONFIG_FILE_CONTENTS: &str = include_str!("../darkwiki.toml");
-pub const DOCS_PATH: &str = "~/darkwiki";
-
-/// darkwiki cli
-#[derive(Clone, Debug, Deserialize, StructOpt, StructOptToml)]
-#[serde(default)]
-#[structopt(name = "darkwiki")]
-pub struct Args {
-    /// Sets a custom config file
-    #[structopt(long)]
-    pub config: Option<String>,
-    /// Sets Datastore Path
-    #[structopt(long, default_value = "~/.config/darkfi/darkwiki")]
-    pub datastore: String,
-    /// JSON-RPC listen URL
-    #[structopt(long = "rpc", default_value = "tcp://127.0.0.1:13055")]
-    pub rpc_listen: Url,
-    #[structopt(flatten)]
-    pub net: SettingsOpt,
-    /// Increase verbosity
-    #[structopt(short, parse(from_occurrences))]
-    pub verbose: u8,
-}
-
-fn on_receive_operation(op: Operation, datastore_path: &Path) -> DarkWikiResult<()> {
-    let json_files_path = datastore_path.join("files");
-    let docs_path = PathBuf::from(expand_path(&DOCS_PATH)?);
-
-    let id_path = json_files_path.join(op.id());
-
-    let json_file = load_json_file::<Sequence>(&id_path);
-    let mut seq: Sequence = if let Ok(file) = json_file { file } else { Sequence::new(&op.id()) };
-
-    seq.add_op(&op)?;
-    save_json_file::<Sequence>(&id_path, &seq)?;
-
-    //let st = seq.apply();
-    //save_file(&docs_path.join(op.id()), &st)?;
-
-    Ok(())
-}
-
-fn on_receive_update(datastore_path: &Path) -> DarkWikiResult<Vec<Operation>> {
-    let ret = vec![];
-
-    let json_files_path = datastore_path.join("files");
-    let docs_path = PathBuf::from(expand_path(&DOCS_PATH)?);
-
-    let files = read_dir(&docs_path).unwrap();
-
-    for file in files {
-        let file_path = file.unwrap().path();
-        let path = docs_path.join(&file_path);
-        let edit = load_file(&path)?;
-
-        if let Ok(seq) = load_json_file::<Sequence>(&json_files_path.join(&file_path)) {
-            //
-            // TODO the transformation should happen here
-            //
-        } else {
-            let mut seq = Sequence::new(&gen_id(30));
-            //seq.insert(0, &edit)?;
-            save_json_file(&json_files_path.join(file_path), &seq)?;
-        }
-    }
-
-    Ok(ret)
-}
-
-async fn start(
-    update_notifier_rv: async_channel::Receiver<()>,
-    raft_sender: async_channel::Sender<Operation>,
-    raft_receiver: async_channel::Receiver<Operation>,
-    datastore_path: PathBuf,
-) -> DarkWikiResult<()> {
-    loop {
-        select! {
-            _ = update_notifier_rv.recv().fuse() => {
-                let ops = on_receive_update(&datastore_path)?;
-                for op in ops {
-                    raft_sender.send(op).await.map_err(Error::from)?;
-                }
-            }
-            op = raft_receiver.recv().fuse() => {
-                let op = op.map_err(Error::from)?;
-                on_receive_operation(op, &datastore_path)?;
-            }
-
-        }
-    }
-}
-
-async_daemonize!(realmain);
-async fn realmain(settings: Args, executor: Arc<Executor<'_>>) -> Result<()> {
-    let datastore_path = expand_path(&settings.datastore)?;
-
-    create_dir_all(expand_path(&DOCS_PATH)?)?;
-    create_dir_all(datastore_path.join("files"))?;
-
-    let (update_notifier_sx, update_notifier_rv) = async_channel::unbounded::<()>();
-
-    //
-    // RPC
-    //
-    let rpc_interface = Arc::new(JsonRpcInterface::new(update_notifier_sx));
-    executor.spawn(listen_and_serve(settings.rpc_listen.clone(), rpc_interface)).detach();
-
-    //
-    // Raft
-    //
-    let net_settings = settings.net;
-    let seen_net_msgs = Arc::new(Mutex::new(FxHashMap::default()));
-
-    let datastore_raft = datastore_path.join("darkwiki.db");
-    let raft_settings = RaftSettings { datastore_path: datastore_raft, ..RaftSettings::default() };
-
-    let mut raft = Raft::<Operation>::new(raft_settings, seen_net_msgs.clone())?;
-
-    executor
-        .spawn(start(update_notifier_rv, raft.sender(), raft.receiver(), datastore_path.clone()))
-        .detach();
-
-    //
-    // P2p setup
-    //
-    let (p2p_send_channel, p2p_recv_channel) = async_channel::unbounded::<NetMsg>();
-
-    let p2p = net::P2p::new(net_settings.into()).await;
-    let p2p = p2p.clone();
-
-    let registry = p2p.protocol_registry();
-
-    let raft_node_id = raft.id();
-    registry
-        .register(net::SESSION_ALL, move |channel, p2p| {
-            let raft_node_id = raft_node_id.clone();
-            let sender = p2p_send_channel.clone();
-            let seen_net_msgs_cloned = seen_net_msgs.clone();
-            async move {
-                ProtocolRaft::init(raft_node_id, channel, sender, p2p, seen_net_msgs_cloned).await
-            }
-        })
-        .await;
-
-    p2p.clone().start(executor.clone()).await?;
-
-    executor.spawn(p2p.clone().run(executor.clone())).detach();
-
-    //
-    // Waiting Exit signal
-    //
-    let (signal, shutdown) = async_channel::bounded::<()>(1);
-    ctrlc_async::set_async_handler(async move {
-        warn!(target: "darkwiki", "Catch exit signal");
-        // cleaning up tasks running in the background
-        if let Err(e) = signal.send(()).await {
-            error!("Error on sending exit signal: {}", e);
-        }
-    })
-    .unwrap();
-
-    raft.run(p2p.clone(), p2p_recv_channel.clone(), executor.clone(), shutdown.clone()).await?;
-
-    Ok(())
-}

+ 0 - 195
script/research/darkwiki/src/sequence.rs

@@ -1,195 +0,0 @@
-use std::{io, result::Result};
-
-use serde::{Deserialize, Serialize};
-use unicode_segmentation::UnicodeSegmentation;
-
-use darkfi::util::serial::{Decodable, Encodable, SerialDecodable, SerialEncodable};
-
-use crate::error::DarkWikiError;
-
-#[derive(PartialEq, Serialize, Deserialize, Clone, Debug)]
-pub enum OperationMethod {
-    Delete(u64),
-    Insert(String),
-    Retain(u64),
-}
-
-#[derive(PartialEq, Serialize, Deserialize, SerialEncodable, SerialDecodable, Clone, Debug)]
-pub struct Operation {
-    id: String,
-    method: OperationMethod,
-}
-
-impl Operation {
-    pub fn id(&self) -> String {
-        self.id.clone()
-    }
-}
-
-#[derive(PartialEq, Serialize, Deserialize, Clone, Debug)]
-pub struct Sequence {
-    id: String,
-    operations: Vec<OperationMethod>,
-    len: u64,
-}
-
-impl Sequence {
-    pub fn new(id: &str) -> Self {
-        Self { id: id.to_string(), operations: vec![], len: 0 }
-    }
-    ///
-    /// Apply all operations to the provided &str
-    /// Return the final String
-    ///
-    pub fn apply(&self, s: &str) -> String {
-        let mut st = vec![];
-        let mut chars = s.graphemes(true).collect::<Vec<&str>>();
-        for op in &self.operations {
-            match op {
-                OperationMethod::Retain(n) => {
-                    st.extend(chars[..(*n as usize)].to_vec());
-                }
-                OperationMethod::Delete(n) => {
-                    chars.drain(0..(*n as usize));
-                }
-                OperationMethod::Insert(insert) => {
-                    st.extend(insert.graphemes(true).collect::<Vec<&str>>());
-                }
-            }
-        }
-
-        st.join("")
-    }
-
-    ///
-    /// Add new operation
-    /// Return AddOperationFailed error if failed
-    ///
-    pub fn add_op(&mut self, op: &Operation) -> Result<(), DarkWikiError> {
-        match &op.method {
-            OperationMethod::Delete(n) => {
-                if *n == 0 {
-                    return Ok(())
-                }
-            }
-            OperationMethod::Insert(insert) => {
-                if insert.is_empty() {
-                    return Ok(())
-                }
-            }
-            OperationMethod::Retain(n) => {
-                if *n == 0 {
-                    return Ok(())
-                }
-            }
-        }
-
-        self.operations.push(op.method.clone());
-        Ok(())
-    }
-
-    ///
-    /// Insert string at `n` position with Insert Operation
-    /// Return AddOperationFailed if failed
-    ///
-    pub fn insert(&mut self, st: &str) -> Result<Operation, DarkWikiError> {
-        let method = OperationMethod::Insert(st.into());
-        let op = Operation { id: self.id.clone(), method };
-        self.add_op(&op)?;
-        Ok(op)
-    }
-
-    ///
-    /// Move the position of cursor
-    /// Return AddOperationFailed if failed
-    ///
-    pub fn retain(&mut self, n: u64) -> Result<Operation, DarkWikiError> {
-        let method = OperationMethod::Retain(n);
-        let op = Operation { id: self.id.clone(), method };
-        self.add_op(&op)?;
-        Ok(op)
-    }
-
-    ///
-    /// Delete string at `n` position with Delete Operation
-    /// Return AddOperationFailed if failed
-    ///
-    pub fn delete(&mut self, n: u64) -> Result<Operation, DarkWikiError> {
-        let method = OperationMethod::Delete(n);
-        let op = Operation { id: self.id.clone(), method };
-        self.add_op(&op)?;
-        Ok(op)
-    }
-}
-
-impl Encodable for OperationMethod {
-    fn encode<S: io::Write>(&self, mut s: S) -> darkfi::Result<usize> {
-        let len: usize = match self {
-            Self::Delete(i) => (0 as u8).encode(&mut s)? + i.encode(&mut s)?,
-            Self::Insert(t) => (1 as u8).encode(&mut s)? + t.encode(&mut s)?,
-            Self::Retain(i) => (2 as u8).encode(&mut s)? + i.encode(&mut s)?,
-        };
-        Ok(len)
-    }
-}
-
-impl Decodable for OperationMethod {
-    fn decode<D: io::Read>(mut d: D) -> darkfi::Result<Self> {
-        let com: u8 = Decodable::decode(&mut d)?;
-        match com {
-            0 => {
-                let i: u64 = Decodable::decode(&mut d)?;
-                Ok(Self::Delete(i))
-            }
-            1 => {
-                let t: String = Decodable::decode(d)?;
-                Ok(Self::Insert(t))
-            }
-            2 => {
-                let i: u64 = Decodable::decode(&mut d)?;
-                Ok(Self::Retain(i))
-            }
-            _ => Err(darkfi::Error::ParseFailed("Parse OperationMethod failed")),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use darkfi::util::{
-        gen_id,
-        serial::{deserialize, serialize},
-    };
-
-    #[test]
-    fn test_seq() {
-        //
-        // English
-        //
-        let _t = "this is the first paragraph";
-        let mut seq = Sequence::new(&gen_id(30));
-
-        //
-        // Korean
-        //
-        let _t = "안녕하십니까";
-        let mut seq = Sequence::new(&gen_id(30));
-
-        //
-        // Arabic
-        //
-        let _t = "عربي";
-        let mut seq = Sequence::new(&gen_id(30));
-    }
-
-    #[test]
-    fn test_serialize() {
-        let op_method = OperationMethod::Delete(3);
-
-        let op_method_ser = serialize(&op_method);
-        let op_method_deser = deserialize(&op_method_ser).unwrap();
-
-        assert_eq!(op_method, op_method_deser);
-    }
-}