/* This file is part of DarkFi (https://dark.fi) * * Copyright (C) 2020-2023 Dyne.org foundation * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ use std::{ collections::HashMap, fs::{create_dir_all, read_dir, remove_file}, io::stdin, path::{Path, PathBuf}, process::exit, }; use async_std::{ stream::StreamExt, sync::{Arc, Mutex, RwLock}, task, }; use dryoc::classic::crypto_secretbox::{crypto_secretbox_keygen, Key}; use futures::{select, FutureExt}; use lazy_static::lazy_static; use log::{debug, error, info, warn}; use signal_hook::consts::{SIGHUP, SIGINT, SIGQUIT, SIGTERM}; use signal_hook_async_std::Signals; use structopt_toml::{serde::Deserialize, structopt::StructOpt, StructOptToml}; use url::Url; use darkfi::{ async_daemonize, cli_desc, net, raft::{NetMsg, ProtocolRaft, Raft, RaftSettings}, rpc::server::listen_and_serve, util::{ file::{load_file, load_json_file, save_file, save_json_file}, path::{expand_path, get_config_path}, }, Result, }; mod jsonrpc; use jsonrpc::JsonRpcInterface; mod lcs; use lcs::Lcs; mod patch; use patch::{EncryptedPatch, OpMethod, Patch}; mod util; use util::{decrypt_patch, encrypt_patch, get_docs_paths, parse_workspaces, path_to_id}; type Patches = (Vec, Vec, Vec, Vec); lazy_static! { /// This is where we hold our workspaces, so we are also able to refresh them on SIGHUP. static ref WORKSPACES: RwLock> = RwLock::new(HashMap::new()); } pub const CONFIG_FILE: &str = "darkwikid_config.toml"; pub const CONFIG_FILE_CONTENTS: &str = include_str!("../darkwikid_config.toml"); const SYNC_ID_PATH: &str = "sync"; const LOCAL_ID_PATH: &str = "local"; #[derive(Clone, Debug, Deserialize, StructOpt, StructOptToml)] #[serde(default)] #[structopt(name = "darkwikid", about = cli_desc!())] struct Args { /// Increase verbosity (-vvv supported) #[structopt(short, parse(from_occurrences))] verbose: u8, /// Configuration file to use #[structopt(short, long)] config: Option, /// Workspace configuration (repeatable flag) #[structopt(short, long)] workspace: Vec, /// Path where to store wiki's files #[structopt(short, long, default_value = "~/darkwiki")] docs: String, /// Sets author's name for patches #[structopt(long, default_value = "Anonymous")] author: String, /// Generate a new secret for a workspace #[structopt(long)] gen_secret: bool, /// JSON-RPC listen URL #[structopt(long, default_value = "tcp://localhost:24330")] rpc_listen: Url, /// Network settings #[structopt(flatten)] net: net::settings::SettingsOpt, } /// Settings struct used to hold some metadata for DarkWiki struct DarkWikiSettings { author: String, docs_path: PathBuf, store_path: PathBuf, } /// DarkWiki object struct DarkWiki { settings: DarkWikiSettings, #[allow(clippy::type_complexity)] rpc: ( smol::channel::Sender>>, smol::channel::Receiver<(String, bool, Vec)>, ), raft: (smol::channel::Sender, smol::channel::Receiver), } impl DarkWiki { async fn start(&self) -> Result<()> { loop { select! { val = self.rpc.1.recv().fuse() => { let (cmd, dry, files) = match val { Ok(v) => v, Err(e) => { error!("Failed unwrapping val received from RPC: {}", e); continue } }; match cmd.as_str() { "update" => { if let Err(e) = self.on_receive_update(dry, files).await { error!("on_receive_update returned error: {}", e); continue } } "restore" => { if let Err(e) = self.on_receive_restore(dry, files).await { error!("on_receive_restore returned error: {}", e); continue } } x => { warn!("Received unsupported command: {}", x); continue } } } patch = self.raft.1.recv().fuse() => { let patch = match patch { Ok(v) => v, Err(e) => { error!("Failed unwrapping patch received from raft: {}", e); continue } }; for (workspace, key) in WORKSPACES.read().await.iter() { if let Ok(mut patch) = decrypt_patch(&patch, key) { info!("[{}] Receive a {:?}", workspace, patch); patch.workspace = workspace.clone(); if let Err(e) = self.on_receive_patch(&patch) { error!("on_receive_patch returned error: {}", e); } } } } } } } fn on_receive_patch(&self, received_patch: &Patch) -> Result<()> { let sync_id_path = self.settings.store_path.join(SYNC_ID_PATH).join(&received_patch.id); let local_id_path = self.settings.store_path.join(LOCAL_ID_PATH).join(&received_patch.id); if let Ok(mut sync_patch) = load_json_file::(&sync_id_path) { if sync_patch.timestamp == received_patch.timestamp { return Ok(()) } if let Ok(local_patch) = load_json_file::(&local_id_path) { if local_patch.timestamp == sync_patch.timestamp { sync_patch.base = local_patch.to_string(); sync_patch.set_ops(received_patch.ops()); } else { sync_patch.extend_ops(received_patch.ops()); } } sync_patch.timestamp = received_patch.timestamp; sync_patch.author = received_patch.author.clone(); save_json_file::(&sync_id_path, &sync_patch)?; } else if !received_patch.base.is_empty() { save_json_file::(&sync_id_path, received_patch)?; } Ok(()) } async fn on_receive_update(&self, dry: bool, files: Vec) -> Result<()> { let (mut local, mut sync, mut merge) = (vec![], vec![], vec![]); for (workspace, key) in WORKSPACES.read().await.iter() { let (patches, l, s, m) = self.update( dry, &self.settings.docs_path.join(workspace), files.clone(), workspace, )?; local.extend(l); sync.extend(s); merge.extend(m); if !dry { for patch in patches { info!("Send a {:?}", patch); let encrypt_patch = encrypt_patch(&patch, key)?; self.raft.0.send(encrypt_patch).await?; } } } self.rpc.0.send(vec![local, sync, merge]).await?; Ok(()) } async fn on_receive_restore(&self, dry: bool, filenames: Vec) -> Result<()> { let mut patches = vec![]; for (workspace, _) in WORKSPACES.read().await.iter() { patches.extend(self.restore( dry, &self.settings.docs_path.join(workspace), &filenames, workspace, )?); } self.rpc.0.send(vec![patches]).await?; Ok(()) } fn restore( &self, dry: bool, docs_path: &Path, filenames: &[String], workspace: &str, ) -> Result> { let local_path = self.settings.store_path.join(LOCAL_ID_PATH); let mut patches = vec![]; let local_files = read_dir(&local_path)?; for file in local_files { let file_id = file?.file_name(); let file_path = local_path.join(&file_id); let local_patch: Patch = load_json_file(&file_path)?; if local_patch.workspace != workspace { continue } // TODO: FIXME: Simplify this logic, what is this? Add comments. if !filenames.is_empty() && !filenames.contains(&local_patch.path.to_string()) { continue } if let Ok(doc) = load_file(&docs_path.join(&local_patch.path)) { if local_patch.to_string() == doc { continue } } if !dry { self.save_doc(&local_patch.path, &local_patch.to_string(), workspace)?; } patches.push(local_patch); } Ok(patches) } // TODO: Add debug/info statements and refactor this function, there's too many things going on here. fn update( &self, dry: bool, docs_path: &Path, filenames: Vec, workspace: &str, ) -> Result { let (mut patches, mut local_patches, mut sync_patches, mut merge_patches) = (vec![], vec![], vec![], vec![]); let local_path = self.settings.store_path.join(LOCAL_ID_PATH); let sync_path = self.settings.store_path.join(SYNC_ID_PATH); // Save and compare docs in darkwiki and local dirs, then // merge with sync patches if any have been received. let mut docs = vec![]; get_docs_paths(&mut docs, docs_path, None)?; for doc in docs { let doc_path = doc.to_str().unwrap(); // FIXME: IDGI if !filenames.is_empty() && !filenames.contains(&doc_path.to_string()) { continue } // Load doc content let edit = load_file(&docs_path.join(doc_path))?; if edit.is_empty() { continue } let doc_id = path_to_id(doc_path, workspace); // Create new patch let mut new_patch = Patch::new(doc_path, &doc_id, &self.settings.author, workspace); // Check for any changes found with local doc and darkwiki doc if let Ok(local_patch) = load_json_file::(&local_path.join(&doc_id)) { // No changes found if local_patch.to_string() == edit { continue } // Check the differences with LCS algorithm let local_patch_str = local_patch.to_string(); let lcs = Lcs::new(&local_patch_str, &edit); let lcs_ops = lcs.ops(); // Add the change ops to the new patch for op in lcs_ops { new_patch.add_op(&op); } new_patch.base = local_patch.to_string(); local_patches.push(new_patch.clone()); let mut b_patch = new_patch.clone(); b_patch.base = "".to_string(); patches.push(b_patch); // Check if the same doc has received a patch from the network if let Ok(sync_patch) = load_json_file::(&sync_path.join(&doc_id)) { if !Self::is_delete_patch(&sync_patch) { if sync_patch.timestamp != local_patch.timestamp { sync_patches.push(sync_patch.clone()); let sync_patch_t = new_patch.transform(&sync_patch); new_patch = new_patch.merge(&sync_patch_t); if !dry { self.save_doc(doc_path, &new_patch.to_string(), workspace)?; } merge_patches.push(new_patch.clone()); } } else { merge_patches.push(sync_patch); patches = vec![]; } } } else { new_patch.base = edit.to_string(); local_patches.push(new_patch.clone()); patches.push(new_patch.clone()); }; if !dry { save_json_file(&local_path.join(&doc_id), &new_patch)?; save_json_file(&sync_path.join(&doc_id), &new_patch)?; } } // Check if a new patch is received and save the new changes // in both local and darkwiki dirs. let sync_files = read_dir(&sync_path)?; for file in sync_files { let file_id = file?.file_name(); let file_path = sync_path.join(&file_id); let sync_patch: Patch = load_json_file(&file_path)?; if sync_patch.workspace != workspace { continue } if Self::is_delete_patch(&sync_patch) { if local_path.join(&sync_patch.id).exists() { sync_patches.push(sync_patch.clone()); } if !dry { remove_file(docs_path.join(&sync_patch.path))?; remove_file(local_path.join(&sync_patch.id))?; remove_file(file_path)?; } continue } if let Ok(local_patch) = load_json_file::(&local_path.join(&file_id)) { if local_patch.timestamp == sync_patch.timestamp { continue } } // TODO: FIXME: IDGI AGAIN, HALP if !filenames.is_empty() && !filenames.contains(&sync_patch.path.to_string()) { continue } if !dry { self.save_doc(&sync_patch.path, &sync_patch.to_string(), workspace)?; save_json_file(&local_path.join(file_id), &sync_patch)?; } if !sync_patches.contains(&sync_patch) { sync_patches.push(sync_patch); } } // Check if any doc is removed from darkwiki filesystem. let local_files = read_dir(&local_path)?; for file in local_files { let file_id = file?.file_name(); let file_path = local_path.join(&file_id); let local_patch: Patch = load_json_file(&file_path)?; if local_patch.workspace != workspace { continue } // TODO: FIXME: Is it just supposed to check that filenames doesn't contain the local_patch? if !filenames.is_empty() && !filenames.contains(&local_patch.path.to_string()) { continue } if !docs_path.join(&local_patch.path).exists() { let mut new_patch = Patch::new( &local_patch.path, &local_patch.id, &self.settings.author, &local_patch.workspace, ); new_patch.add_op(&OpMethod::Delete(local_patch.to_string().len() as u64)); patches.push(new_patch.clone()); new_patch.base = local_patch.base; local_patches.push(new_patch); if !dry { remove_file(file_path)?; } } } Ok((patches, local_patches, sync_patches, merge_patches)) } fn save_doc(&self, path: &str, edit: &str, workspace: &str) -> Result<()> { let path = self.settings.docs_path.join(workspace).join(path); if let Some(p) = path.parent() { if !p.exists() && !p.to_str().unwrap().is_empty() { create_dir_all(p)?; } } save_file(&path, edit) } fn is_delete_patch(patch: &Patch) -> bool { if patch.ops().0.len() != 1 { return false } if let OpMethod::Delete(d) = patch.ops().0[0] { if patch.base.len() as u64 == d { return true } } false } } async fn handle_signals( mut signals: Signals, cfg_path: PathBuf, term_tx: smol::channel::Sender<()>, ) { debug!("Started signal handler"); while let Some(signal) = signals.next().await { match signal { SIGHUP => { info!("Caught SIGHUP"); let toml_contents = match std::fs::read_to_string(cfg_path.clone()) { Ok(v) => v, Err(e) => { error!("Couldn't load configuration file: {}", e); continue } }; *WORKSPACES.write().await = parse_workspaces(&toml_contents); info!("Reloaded workspaces"); } SIGTERM | SIGINT | SIGQUIT => { term_tx.send(()).await.unwrap(); } _ => unreachable!(), } } } async_daemonize!(realmain); async fn realmain(args: Args, executor: Arc>) -> Result<()> { let cfg_path = get_config_path(args.config, CONFIG_FILE)?; let docs_path = expand_path(&args.docs)?; let store_path = expand_path(docs_path.join(".log").to_str().unwrap())?; create_dir_all(docs_path.clone())?; create_dir_all(store_path.clone())?; create_dir_all(store_path.join(LOCAL_ID_PATH))?; create_dir_all(store_path.join(SYNC_ID_PATH))?; if args.gen_secret { eprintln!("Generating a new workspace"); loop { eprint!("Input the name for the new workspace (use ascii chars): "); let mut workspace = String::new(); stdin().read_line(&mut workspace)?; // Non-exhaustive let workspace = workspace.replace(['\t', '\r', ' ', '/', '\\', '\'', '&', '~', ':'], "_"); if workspace.is_empty() || workspace.len() < 3 { eprintln!("Error: Workspace name is empty or less than 3 characters. Try again."); continue } let secret = bs58::encode(crypto_secretbox_keygen()).into_string(); create_dir_all(docs_path.join(workspace.clone()))?; println!("Created workspace: {}:{}", workspace, secret); eprintln!("Please add it to the config file."); return Ok(()) } } // Signal handling for config reload and graceful termination. let signals = Signals::new([SIGHUP, SIGTERM, SIGINT, SIGQUIT])?; let handle = signals.handle(); let (term_tx, term_rx) = smol::channel::bounded::<()>(1); let signals_task = task::spawn(handle_signals(signals, cfg_path.clone(), term_tx)); info!("Set up signal handling"); { info!("Parsing configuration file for workspaces"); let toml_contents = std::fs::read_to_string(cfg_path.clone())?; *WORKSPACES.write().await = parse_workspaces(&toml_contents); if WORKSPACES.read().await.is_empty() { eprintln!("Please add atleast one workspace to the config file."); eprintln!("Run \"$ darkwikid --gen-secret\" to create a new workspace."); exit(1); } } let (rpc_tx, rpc_rx) = smol::channel::unbounded::<(String, bool, Vec)>(); let (notify_tx, notify_rx) = smol::channel::unbounded::>>(); // =============== // JSON-RPC server // =============== let rpc_iface = Arc::new(JsonRpcInterface::new(rpc_tx, notify_rx)); let _ex = executor.clone(); executor.spawn(listen_and_serve(args.rpc_listen, rpc_iface, _ex)).detach(); // ==== // Raft // ==== let seen_net_msgs = Arc::new(Mutex::new(HashMap::new())); let store_raft = store_path.join("darkwiki.db"); let raft_settings = RaftSettings { datastore_path: store_raft, ..RaftSettings::default() }; // FIXME: This is a bad design, and needs a proper rework. let raft = Arc::new(Mutex::new(Raft::::new(raft_settings, seen_net_msgs.clone())?)); // ========= // P2P setup // ========= let mut net_settings = args.net.clone(); net_settings.app_version = Some(option_env!("CARGO_PKG_VERSION").unwrap_or("").to_string()); let (p2p_tx, p2p_rx) = smol::channel::unbounded::(); let p2p = net::P2p::new(net_settings.into()).await; let registry = p2p.protocol_registry(); let raft_node_id = raft.lock().await.id(); registry.register(net::SESSION_ALL, move | channel, p2p| { let raft_node_id = raft_node_id.clone(); let sender = p2p_tx.clone(); let seen_net_msgs = seen_net_msgs.clone(); async move { ProtocolRaft::init(raft_node_id, channel, sender, p2p, seen_net_msgs).await } }).await; p2p.clone().start(executor.clone()).await?; executor.spawn(p2p.clone().run(executor.clone())).detach(); // ============== // Darkwiki start // ============== let raft_tx = raft.lock().await.sender(); let raft_rx = raft.lock().await.receiver(); executor .spawn(async move { let settings = DarkWikiSettings { author: args.author, store_path, docs_path }; let dw = DarkWiki { settings, raft: (raft_tx, raft_rx), rpc: (notify_tx, rpc_rx) }; dw.start().await.unwrap(); }) .detach(); let (raft_term_tx, raft_term_rx) = smol::channel::bounded::<()>(1); let _p2p = p2p.clone(); let _ex = executor.clone(); executor .spawn(async move { raft.lock().await.run(_p2p, p2p_rx, _ex, raft_term_rx).await.unwrap() }) .detach(); // Wait for termination signal term_rx.recv().await?; eprint!("\r"); info!("Caught termination signal, cleaning up and exiting..."); handle.close(); signals_task.await; info!("Stopping Raft..."); raft_term_tx.send(()).await.unwrap(); info!("Stopping P2P network..."); p2p.stop().await; info!("Bye."); Ok(()) }