darkfid: include miners tempaltes new trees when purging unferenced trees

bin/darkfid/src/registry/mod.rs

@@ -17,10 +17,11 @@
  */
 
 use std::{
-    collections::{HashMap, HashSet},
+    collections::{BTreeSet, HashMap, HashSet},
     sync::Arc,
 };
 
+use sled_overlay::sled::IVec;
 use smol::lock::{Mutex, RwLock};
 use tinyjson::JsonValue;
 use tracing::{error, info};
@@ -479,4 +480,16 @@ impl DarkfiMinersRegistry {
 
         Ok(())
     }
+
+    /// Auxilliary function to retrieve all current block templates
+    /// newly opened trees.
+    pub fn new_trees(&self, block_templates: &HashMap<String, BlockTemplate>) -> BTreeSet<IVec> {
+        let mut new_trees = BTreeSet::new();
+        for block_template in block_templates.values() {
+            for new_tree in &block_template.new_trees {
+                new_trees.insert(new_tree.clone());
+            }
+        }
+        new_trees
+    }
 }

bin/darkfid/src/registry/model.rs

@@ -19,6 +19,7 @@
 use std::{collections::HashMap, str::FromStr};
 
 use rand::rngs::OsRng;
+use sled_overlay::sled::IVec;
 use tinyjson::JsonValue;
 use tracing::info;
 
@@ -139,6 +140,8 @@ impl MinerRewardsRecipientConfig {
 pub struct BlockTemplate {
     /// Block that is being mined
     pub block: BlockInfo,
+    /// New `sled` trees opened the overlay this block was generated
+    pub new_trees: Vec<IVec>,
     /// RandomX current and next keys pair
     pub randomx_keys: (HeaderHash, Option<HeaderHash>),
     /// Compacted block mining target
@@ -154,12 +157,13 @@ pub struct BlockTemplate {
 impl BlockTemplate {
     fn new(
         block: BlockInfo,
+        new_trees: Vec<IVec>,
         randomx_keys: (HeaderHash, Option<HeaderHash>),
         target: Vec<u8>,
         difficulty: f64,
         secret: SecretKey,
     ) -> Self {
-        Self { block, randomx_keys, target, difficulty, secret, submitted: false }
+        Self { block, new_trees, randomx_keys, target, difficulty, secret, submitted: false }
     }
 
     pub fn job_notification(&self) -> (String, JsonValue) {
@@ -333,6 +337,7 @@ pub async fn generate_next_block_template(
 
     Ok(BlockTemplate::new(
         next_block,
+        diff.new_trees(),
         randomx_keys,
         target,
         difficulty,

bin/darkfid/src/task/consensus.rs

@@ -29,7 +29,10 @@ use darkfi_serial::serialize_async;
 use tracing::{error, info};
 
 use crate::{
-    task::{garbage_collect_task, sync_task},
+    task::{
+        garbage_collect::{garbage_collect_task, purge_unreferenced_trees},
+        sync_task,
+    },
     DarkfiNodePtr,
 };
 
@@ -190,10 +193,14 @@ async fn consensus_task(
             }
         };
 
+        // Refresh mining registry
         if let Err(e) = node.registry.refresh(&node.validator).await {
             error!(target: "darkfid", "Failed refreshing mining block templates: {e}")
         }
 
+        // Purge all unreferenced contract trees from the database
+        purge_unreferenced_trees(node).await;
+
         if confirmed.is_empty() {
             continue
         }

bin/darkfid/src/task/garbage_collect.rs

@@ -26,11 +26,6 @@ use crate::DarkfiNodePtr;
 pub async fn garbage_collect_task(node: DarkfiNodePtr) -> Result<()> {
     info!(target: "darkfid::task::garbage_collect_task", "Starting garbage collection task...");
 
-    // Purge all unreferenced contract trees from the database
-    if let Err(e) = node.validator.consensus.purge_unreferenced_trees().await {
-        error!(target: "darkfid::task::garbage_collect_task", "Purging unreferenced contract trees from the database failed: {e}");
-    }
-
     // Grab all current unproposed transactions.  We verify them in batches,
     // to not load them all in memory.
     let (mut last_checked, mut txs) =
@@ -116,9 +111,6 @@ pub async fn garbage_collect_task(node: DarkfiNodePtr) -> Result<()> {
                 )
                 .await;
 
-                // Drop new trees opened by the forks' overlay
-                overlay.lock().unwrap().overlay.lock().unwrap().purge_new_trees()?;
-
                 // Check result
                 match result {
                     Ok(_) => valid = true,
@@ -169,11 +161,32 @@ pub async fn garbage_collect_task(node: DarkfiNodePtr) -> Result<()> {
         };
     }
 
-    // Purge all unreferenced contract trees from the database again
-    if let Err(e) = node.validator.consensus.purge_unreferenced_trees().await {
-        error!(target: "darkfid::task::garbage_collect_task", "Purging unreferenced contract trees from the database failed: {e}");
-    }
-
     info!(target: "darkfid::task::garbage_collect_task", "Garbage collection finished successfully!");
     Ok(())
 }
+
+/// Auxiliary function to purge all unreferenced contract trees from
+/// the node database.
+pub async fn purge_unreferenced_trees(node: &DarkfiNodePtr) {
+    // Grab node registry locks
+    let submit_lock = node.registry.submit_lock.write().await;
+    let block_templates = node.registry.block_templates.write().await;
+    let jobs = node.registry.jobs.write().await;
+    let mm_jobs = node.registry.mm_jobs.write().await;
+
+    // Purge all unreferenced contract trees from the database
+    if let Err(e) = node
+        .validator
+        .consensus
+        .purge_unreferenced_trees(&mut node.registry.new_trees(&block_templates))
+        .await
+    {
+        error!(target: "darkfid::task::garbage_collect::purge_unreferenced_trees", "Purging unreferenced contract trees from the database failed: {e}");
+    }
+
+    // Release registry locks
+    drop(block_templates);
+    drop(jobs);
+    drop(mm_jobs);
+    drop(submit_lock);
+}

src/validator/consensus.rs

@@ -624,23 +624,23 @@ impl Consensus {
 
     /// Auxiliary function to purge all unreferenced contract trees
     /// from the database.
-    pub async fn purge_unreferenced_trees(&self) -> Result<()> {
+    pub async fn purge_unreferenced_trees(
+        &self,
+        referenced_trees: &mut BTreeSet<IVec>,
+    ) -> Result<()> {
         // Grab a lock over current forks
         let lock = self.forks.read().await;
 
-        // Keep track of referenced trees
-        let mut referenced_trees = BTreeSet::new();
-
         // Check if we have forks
         if lock.is_empty() {
             // If no forks exist, build a new one so we retrieve the
             // native/protected trees references.
             let fork = Fork::new(self.blockchain.clone(), self.module.read().await.clone()).await?;
-            fork.referenced_trees(&mut referenced_trees);
+            fork.referenced_trees(referenced_trees);
         } else {
             // Iterate over current forks to retrieve referenced trees
             for fork in lock.iter() {
-                fork.referenced_trees(&mut referenced_trees);
+                fork.referenced_trees(referenced_trees);
             }
         }