metrics.rs 57 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378
  1. /* This file is part of DarkFi (https://dark.fi)
  2. *
  3. * Copyright (C) 2020-2026 Dyne.org foundation
  4. *
  5. * This program is free software: you can redistribute it and/or modify
  6. * it under the terms of the GNU Affero General Public License as
  7. * published by the Free Software Foundation, either version 3 of the
  8. * License, or (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU Affero General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU Affero General Public License
  16. * along with this program. If not, see <https://www.gnu.org/licenses/>.
  17. */
  18. use std::{
  19. fmt, slice,
  20. sync::{Arc, Mutex, MutexGuard},
  21. };
  22. use sled_overlay::{sled, SledDbOverlay};
  23. use tracing::{debug, info};
  24. use darkfi::{
  25. blockchain::SledDbOverlayPtr,
  26. util::time::{DateTime, Timestamp},
  27. validator::fees::GasData,
  28. Error, Result,
  29. };
  30. use darkfi_sdk::{num_traits::ToBytes, tx::TransactionHash};
  31. use darkfi_serial::{async_trait, deserialize, serialize, SerialDecodable, SerialEncodable};
  32. /// Gas metrics tree name.
  33. pub const SLED_GAS_METRICS_TREE: &[u8] = b"_gas_metrics";
  34. /// Gas metrics `by_height` tree that contains all metrics by height.
  35. pub const SLED_GAS_METRICS_BY_HEIGHT_TREE: &[u8] = b"_gas_metrics_by_height";
  36. /// Transaction gas data tree name.
  37. pub const SLED_TX_GAS_DATA_TREE: &[u8] = b"_tx_gas_data";
  38. /// The time interval for [`GasMetricsKey`]s in the main tree, specified in seconds.
  39. /// Metrics are stored in hourly intervals (3600 seconds), meaning all metrics accumulated
  40. /// within a specific hour are stored using a key representing the start of that hour.
  41. pub const GAS_METRICS_KEY_TIME_INTERVAL: u64 = 3600;
  42. #[derive(Debug, Clone, Default, Eq, PartialEq, SerialEncodable, SerialDecodable)]
  43. /// Represents metrics used to capture key statistical data.
  44. pub struct Metrics {
  45. /// An aggregate value that represents the sum of the metrics.
  46. pub sum: u64,
  47. /// The smallest value in the series of measured metrics.
  48. pub min: u64,
  49. /// The largest value in the series of measured metrics.
  50. pub max: u64,
  51. }
  52. // Temporarily disable unused warnings until the store is integrated with the explorer
  53. #[allow(dead_code)]
  54. impl Metrics {
  55. /// Constructs a [`Metrics`] instance with provided parameters.
  56. pub fn new(sum: u64, min: u64, max: u64) -> Self {
  57. Self { sum, min, max }
  58. }
  59. }
  60. /// Structure for managing gas metrics across all transactions in the store.
  61. ///
  62. /// This struct maintains running totals, extrema, and transaction counts to efficiently calculate
  63. /// metrics without the need to iterate through previous transactions when new data is added. It is used to build a
  64. /// comprehensive view of gas metrics across the blockchain's history, including total gas, WASM gas,
  65. /// ZK circuit gas, and signature gas. The structure allows for O(1) performance in calculating
  66. /// averages and updating min/max values.
  67. #[derive(Clone, Default, Eq, PartialEq, SerialEncodable, SerialDecodable)]
  68. pub struct GasMetrics {
  69. /// Represents the total count of transactions tracked by the metrics store.
  70. pub txs_count: u64,
  71. /// Overall gas consumed metrics across all transactions.
  72. pub total_gas: Metrics,
  73. /// Gas used across all executed wasm transactions.
  74. pub wasm_gas: Metrics,
  75. /// Gas consumed across all zk circuit computations.
  76. pub zk_circuits_gas: Metrics,
  77. /// Gas used metrics related to signatures across transactions.
  78. pub signatures_gas: Metrics,
  79. /// Gas consumed for deployments across transactions.
  80. pub deployments_gas: Metrics,
  81. /// The time the metrics was calculated
  82. pub timestamp: Timestamp,
  83. }
  84. // Temporarily disable unused warnings until the store is integrated with the explorer
  85. #[allow(dead_code)]
  86. impl GasMetrics {
  87. /// Creates a [`GasMetrics`] instance.
  88. pub fn new(
  89. txs_count: u64,
  90. total_gas: Metrics,
  91. wasm_gas: Metrics,
  92. zk_circuit_gas: Metrics,
  93. signature_gas: Metrics,
  94. deployment_gas: Metrics,
  95. timestamp: Timestamp,
  96. ) -> Self {
  97. Self {
  98. txs_count,
  99. total_gas,
  100. wasm_gas,
  101. zk_circuits_gas: zk_circuit_gas,
  102. signatures_gas: signature_gas,
  103. deployments_gas: deployment_gas,
  104. timestamp,
  105. }
  106. }
  107. /// Provides the average of the total gas used.
  108. pub fn avg_total_gas_used(&self) -> u64 {
  109. self.total_gas.sum.checked_div(self.txs_count).unwrap_or_default()
  110. }
  111. /// Provides the average of the gas used across WASM transactions.
  112. pub fn avg_wasm_gas_used(&self) -> u64 {
  113. self.wasm_gas.sum.checked_div(self.txs_count).unwrap_or_default()
  114. }
  115. /// Provides the average of the gas consumed across Zero-Knowledge Circuit computations.
  116. pub fn avg_zk_circuits_gas_used(&self) -> u64 {
  117. self.zk_circuits_gas.sum.checked_div(self.txs_count).unwrap_or_default()
  118. }
  119. /// Provides the average of the gas used to sign transactions.
  120. pub fn avg_signatures_gas_used(&self) -> u64 {
  121. self.signatures_gas.sum.checked_div(self.txs_count).unwrap_or_default()
  122. }
  123. /// Provides the average of the gas used for deployments.
  124. pub fn avg_deployments_gas_used(&self) -> u64 {
  125. self.deployments_gas.sum.checked_div(self.txs_count).unwrap_or_default()
  126. }
  127. /// Adds new [`GasData`] to the existing accumulated values.
  128. ///
  129. /// This method updates running totals, transaction counts, and min/max values
  130. /// for various gas metric categories. It accumulates new data without reading existing
  131. /// averages, minimums, or maximums from the database to optimize performance.
  132. pub fn add(&mut self, tx_gas_data: &[GasData]) {
  133. for gas_data in tx_gas_data {
  134. // Increment number of transactions included in stats
  135. self.txs_count += 1;
  136. // Update the statistics related to total gas
  137. self.total_gas.sum += gas_data.total_gas_used();
  138. // Update the statistics related to WASM gas
  139. self.wasm_gas.sum += gas_data.wasm;
  140. // Update the statistics related to ZK circuit gas
  141. self.zk_circuits_gas.sum += gas_data.zk_circuits;
  142. // Update the statistics related to signature gas
  143. self.signatures_gas.sum += gas_data.signatures;
  144. // Update the statistics related to deployment gas
  145. self.deployments_gas.sum += gas_data.deployments;
  146. if self.txs_count == 1 {
  147. // For the first transaction, set min/max to the transaction values
  148. self.total_gas.min = gas_data.total_gas_used();
  149. self.total_gas.max = gas_data.total_gas_used();
  150. self.wasm_gas.min = gas_data.wasm;
  151. self.wasm_gas.max = gas_data.wasm;
  152. self.zk_circuits_gas.min = gas_data.zk_circuits;
  153. self.zk_circuits_gas.max = gas_data.zk_circuits;
  154. self.signatures_gas.min = gas_data.signatures;
  155. self.signatures_gas.max = gas_data.signatures;
  156. self.deployments_gas.min = gas_data.deployments;
  157. self.deployments_gas.max = gas_data.deployments;
  158. return;
  159. }
  160. // For subsequent transactions, compare with min/max
  161. self.total_gas.min = self.total_gas.min.min(gas_data.total_gas_used());
  162. self.total_gas.max = self.total_gas.max.max(gas_data.total_gas_used());
  163. self.wasm_gas.min = self.wasm_gas.min.min(gas_data.wasm);
  164. self.wasm_gas.max = self.wasm_gas.max.max(gas_data.wasm);
  165. self.zk_circuits_gas.min = self.zk_circuits_gas.min.min(gas_data.zk_circuits);
  166. self.zk_circuits_gas.max = self.zk_circuits_gas.max.max(gas_data.zk_circuits);
  167. self.signatures_gas.min = self.signatures_gas.min.min(gas_data.signatures);
  168. self.signatures_gas.max = self.signatures_gas.max.max(gas_data.signatures);
  169. self.deployments_gas.min = self.deployments_gas.min.min(gas_data.deployments);
  170. self.deployments_gas.max = self.deployments_gas.max.max(gas_data.deployments);
  171. }
  172. }
  173. }
  174. /// Debug formatting support for [`GasMetrics`] instances to include averages.
  175. impl fmt::Debug for GasMetrics {
  176. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  177. f.debug_struct("GasMetrics")
  178. .field("txs_count", &self.txs_count)
  179. .field("avg_total_gas_used", &self.avg_total_gas_used())
  180. .field("avg_wasm_gas_used", &self.avg_wasm_gas_used())
  181. .field("avg_zk_circuits_gas_used", &self.avg_zk_circuits_gas_used())
  182. .field("avg_signatures_gas_used", &self.avg_signatures_gas_used())
  183. .field("avg_deployments_gas_used", &self.avg_deployments_gas_used())
  184. .field("total_gas", &format_args!("{:?}", self.total_gas))
  185. .field("wasm_gas", &format_args!("{:?}", self.wasm_gas))
  186. .field("zk_circuits_gas", &format_args!("{:?}", self.zk_circuits_gas))
  187. .field("signatures_gas", &format_args!("{:?}", self.signatures_gas))
  188. .field("deployments_gas", &format_args!("{:?}", self.deployments_gas))
  189. .field("timestamp", &self.timestamp)
  190. .finish()
  191. }
  192. }
  193. /// The `MetricStore` serves as the entry point for managing metrics,
  194. /// offering an API for fetching, inserting, and resetting metrics backed by a Sled database.
  195. ///
  196. /// It organizes data into separate Sled trees, including main storage for gas metrics by a defined time interval,
  197. /// a tree containing metrics by height for handling reorgs, and a transaction-specific gas data tree.
  198. /// Different keys, such as gas metric keys, block heights, and transaction hashes, are used to handle
  199. /// various use cases.
  200. ///
  201. /// The `MetricStore` utilizes an overlay pattern for write operations, allowing unified management of metrics,
  202. /// by internally delegating write-related actions like adding metrics and handling reorgs to [`MetricsStoreOverlay`].
  203. #[derive(Clone)]
  204. pub struct MetricsStore {
  205. /// Pointer to the underlying sled database used by the store and its associated overlay
  206. pub sled_db: sled::Db,
  207. /// Primary sled tree for storing gas metrics, utilizing [`GasMetricsKey`] as keys and
  208. /// serialized [`GasMetrics`] as values.
  209. pub main: sled::Tree,
  210. /// Sled tree for storing gas metrics by height, utilizing block `height` as keys
  211. /// and serialized [`GasMetrics`] as values.
  212. pub by_height: sled::Tree,
  213. /// Sled tree for storing transaction gas data, utilizing [`TransactionHash`] inner value as keys
  214. /// and serialized [`GasData`] as values.
  215. pub tx_gas_data: sled::Tree,
  216. }
  217. // Temporarily disable unused warnings until the store is integrated with the explorer
  218. #[allow(dead_code)]
  219. impl MetricsStore {
  220. /// Creates a [`MetricsStore`] instance by opening the necessary
  221. /// trees in the provided sled database [`sled::Db`]
  222. pub fn new(db: &sled::Db) -> Result<Self> {
  223. let main = db.open_tree(SLED_GAS_METRICS_TREE)?;
  224. let tx_gas_data = db.open_tree(SLED_TX_GAS_DATA_TREE)?;
  225. let metrcs_by_height = db.open_tree(SLED_GAS_METRICS_BY_HEIGHT_TREE)?;
  226. Ok(Self { sled_db: db.clone(), main, tx_gas_data, by_height: metrcs_by_height })
  227. }
  228. /// Fetches [`GasMetrics`]s associated with the provided slice of [`GasMetricsKey`]s.
  229. pub fn get(&self, keys: &[GasMetricsKey]) -> Result<Vec<GasMetrics>> {
  230. let mut ret = Vec::with_capacity(keys.len());
  231. for key in keys {
  232. if let Some(metrics_bytes) = self.main.get(key.to_sled_key())? {
  233. let metrics = deserialize(&metrics_bytes).map_err(Error::from)?;
  234. ret.push(metrics);
  235. }
  236. }
  237. Ok(ret)
  238. }
  239. /// Fetches [`GasMetrics`]s associated with the provided slice of [`u32`] heights.
  240. pub fn get_by_height(&self, heights: &[u32]) -> Result<Vec<GasMetrics>> {
  241. let mut ret = Vec::with_capacity(heights.len());
  242. for height in heights {
  243. if let Some(metrics_bytes) = self.by_height.get(height.to_be_bytes())? {
  244. let metrics = deserialize(&metrics_bytes).map_err(Error::from)?;
  245. ret.push(metrics);
  246. }
  247. }
  248. Ok(ret)
  249. }
  250. /// Fetches the most recent [`GasMetrics`] and its associated [`GasMetricsKey`] from the main tree,
  251. /// returning `None` if no metrics are found.
  252. pub fn get_last(&self) -> Result<Option<(GasMetricsKey, GasMetrics)>> {
  253. self.main
  254. .last()?
  255. .map(|(key_bytes, metrics_bytes)| {
  256. // Deserialize gas metrics key and value
  257. let key = GasMetricsKey::from_sled_key(&key_bytes)?;
  258. let metrics: GasMetrics = deserialize(&metrics_bytes).map_err(Error::from)?;
  259. debug!(target: "explorerd::metrics_store::get_last", "Deserialized metrics at key {key}: {metrics:?}");
  260. Ok((key, metrics))
  261. })
  262. .transpose()
  263. }
  264. /// Fetches all [`GasMetrics`] from the main tree without corresponding key, returning an empty `Vec`
  265. /// if no metrics are found.
  266. pub fn get_all_metrics(&self) -> Result<Vec<GasMetrics>> {
  267. // Iterate through all metrics, deserialize each one, and collect results
  268. self.main
  269. .iter()
  270. .map(|iter_result| match iter_result {
  271. Ok((_, metrics_bytes)) => deserialize(&metrics_bytes).map_err(Error::from),
  272. Err(e) => Err(Error::from(e)),
  273. })
  274. .collect()
  275. }
  276. /// Fetches the most recent [`GasMetrics`] and its associated `height` from the `by_height` tree, returning `None` if no metrics are found.
  277. pub fn get_last_by_height(&self) -> Result<Option<(u32, GasMetrics)>> {
  278. self.by_height
  279. .last()?
  280. .map(|(height_bytes, metrics_bytes)| {
  281. // Deserialize height key and value
  282. let key_bytes: [u8; 4] = height_bytes.as_ref().try_into().unwrap();
  283. let height = u32::from_be_bytes(key_bytes);
  284. let metrics: GasMetrics = deserialize(&metrics_bytes).map_err(Error::from)?;
  285. debug!(target: "explorerd::metrics_store::get_last_by_height", "Deserialized metrics at height {height:?}: {metrics:?}");
  286. Ok((height, metrics))
  287. })
  288. .transpose()
  289. }
  290. /// Fetches the [`GasData`] associated with the provided [`TransactionHash`], or `None` if no gas data is found.
  291. pub fn get_tx_gas_data(&self, tx_hash: &TransactionHash) -> Result<Option<GasData>> {
  292. // Query transaction gas data tree using provided hash
  293. let opt = self.tx_gas_data.get(tx_hash.inner())?;
  294. // Deserialize gas data, map error if needed, return result
  295. opt.map(|value| deserialize(&value).map_err(Error::from)).transpose()
  296. }
  297. /// Adds gas metrics for a specific block of transactions to the store.
  298. ///
  299. /// This function takes block `height`, [`Timestamp`], with associated pairs of [`TransactionHash`] and [`GasData`],
  300. /// and updates the accumulated gas metrics in the store. It handles the storage of metrics for both regular use and
  301. /// blockchain reorganizations.
  302. ///
  303. /// Delegates operation to [`MetricsStoreOverlay::insert_gas_metrics`], whose documentation
  304. /// provides more details.
  305. pub fn insert_gas_metrics(
  306. &self,
  307. block_height: u32,
  308. block_timestamp: &Timestamp,
  309. tx_hashes: &[TransactionHash],
  310. tx_gas_data: &[GasData],
  311. ) -> Result<GasMetricsKey> {
  312. let overlay = MetricsStoreOverlay::new(self.sled_db.clone())?;
  313. overlay.insert_gas_metrics(block_height, block_timestamp, tx_hashes, tx_gas_data)
  314. }
  315. /// Resets the gas metrics in the store to a specified `height` [`u32`].
  316. ///
  317. /// This function reverts all gas metrics data after the given height, effectively
  318. /// undoing changes made beyond that point. It's useful for handling blockchain
  319. /// reorganizations.
  320. ///
  321. /// Delegates operation to [`MetricsStoreOverlay::reset_gas_metrics`], whose documentation
  322. /// provides more details.
  323. pub fn reset_gas_metrics(&self, height: u32) -> Result<()> {
  324. let overlay = MetricsStoreOverlay::new(self.sled_db.clone())?;
  325. overlay.reset_gas_metrics(height)
  326. }
  327. /// Checks if provided [`GasMetricsKey`] exists in the store's main tree.
  328. pub fn contains(&self, key: &GasMetricsKey) -> Result<bool> {
  329. Ok(self.main.contains_key(key.to_sled_key())?)
  330. }
  331. /// Provides the number of stored metrics in the main tree.
  332. pub fn len(&self) -> usize {
  333. self.main.len()
  334. }
  335. /// Provides the number of stored metrics by height.
  336. pub fn len_by_height(&self) -> usize {
  337. self.by_height.len()
  338. }
  339. /// Returns the number of transaction gas usage metrics stored.
  340. pub fn len_tx_gas_data(&self) -> usize {
  341. self.tx_gas_data.len()
  342. }
  343. /// Checks if there are any gas metrics stored.
  344. pub fn is_empty(&self) -> bool {
  345. self.main.is_empty()
  346. }
  347. /// Checks if transaction gas data metrics are stored.
  348. pub fn is_empty_tx_gas_data(&self) -> bool {
  349. self.tx_gas_data.is_empty()
  350. }
  351. }
  352. /// The `MetricsStoreOverlay` provides write operations for managing metrics in conjunction with the
  353. /// underlying sled database. It supports inserting new [`GasData`] into the stored accumulated metrics,
  354. /// adding transaction gas data, and reverting metric changes after a specified height.
  355. struct MetricsStoreOverlay {
  356. /// Pointer to the overlay used for accessing and performing database write operations to the store.
  357. overlay: SledDbOverlayPtr,
  358. /// Pointer managed by the [`MetricsStore`] that references the sled instance on which the overlay operates.
  359. db: sled::Db,
  360. }
  361. impl MetricsStoreOverlay {
  362. /// Instantiate a [`MetricsStoreOverlay`] over the provided
  363. /// [`sled::Db`] instance.
  364. pub fn new(db: sled::Db) -> Result<Self> {
  365. // Create overlay pointer
  366. let overlay = Arc::new(Mutex::new(SledDbOverlay::new(&db, vec![])));
  367. // Open trees
  368. overlay.lock().unwrap().open_tree(SLED_GAS_METRICS_TREE, true)?;
  369. overlay.lock().unwrap().open_tree(SLED_GAS_METRICS_BY_HEIGHT_TREE, true)?;
  370. overlay.lock().unwrap().open_tree(SLED_TX_GAS_DATA_TREE, true)?;
  371. Ok(Self { overlay: overlay.clone(), db })
  372. }
  373. /// Adds the provided [`TransactionHash`] and [`GasData`] pairs to the accumulated [`GasMetrics`]
  374. /// in the store's [`SLED_GAS_METRICS_BY_HEIGHT_TREE`] and [`SLED_GAS_METRICS_TREE`] trees, while
  375. /// also storing transaction gas data in the [`SLED_TX_GAS_DATA_TREE`], committing all changes upon success.
  376. ///
  377. /// This function retrieves the latest recorded metrics, updates them with the new gas data, and
  378. /// stores the accumulated result. It uses the provided `block_timestamp` to create a normalied time-sequenced
  379. /// [`GasMetricsKey`] for metrics storage. The `block_height` is used as a key to store metrics by height
  380. /// which are used to handle chain reorganizations. After updating the aggregate metrics, it stores
  381. /// the transaction gas data for each transaction in the block.
  382. ///
  383. /// Returns the created [`GasMetricsKey`] that can be used to retrieve the metric upon success.
  384. pub fn insert_gas_metrics(
  385. &self,
  386. block_height: u32,
  387. block_timestamp: &Timestamp,
  388. tx_hashes: &[TransactionHash],
  389. tx_gas_data: &[GasData],
  390. ) -> Result<GasMetricsKey> {
  391. // Ensure lengths of tx_hashes and gas_data arrays match
  392. if tx_hashes.len() != tx_gas_data.len() {
  393. return Err(Error::Custom(String::from(
  394. "The lengths of tx_hashes and gas_data arrays must match",
  395. )));
  396. }
  397. // Ensure gas data is provided
  398. if tx_gas_data.is_empty() {
  399. return Err(Error::Custom(String::from("No transaction gas data was provided")));
  400. }
  401. // Lock the database
  402. let mut lock = self.overlay.lock().unwrap();
  403. // Retrieve latest recorded metrics, returning default if not exist
  404. let mut metrics = match self.get_last_by_height(&mut lock)? {
  405. None => GasMetrics::default(),
  406. Some((_, metrics)) => metrics,
  407. };
  408. // Update the accumulated metrics with the provided transaction gas data
  409. metrics.add(tx_gas_data);
  410. // Update the time that the metrics was recorded
  411. metrics.timestamp = *block_timestamp;
  412. // Insert metrics by height
  413. self.insert_by_height(&[block_height], &[metrics.clone()], &mut lock)?;
  414. // Create metrics key based on block_timestamp
  415. let metrics_key = GasMetricsKey::new(block_timestamp)?;
  416. // Normalize metric timestamp based on the key's time interval
  417. metrics.timestamp = GasMetricsKey::normalize_timestamp(block_timestamp)?;
  418. // Insert the gas metrics using metrics key
  419. self.insert(slice::from_ref(&metrics_key), &[metrics], &mut lock)?;
  420. // Insert the transaction gas data for each transaction in the block
  421. self.insert_tx_gas_data(tx_hashes, tx_gas_data, &mut lock)?;
  422. // Commit the changes
  423. lock.apply()?;
  424. Ok(metrics_key)
  425. }
  426. /// Inserts [`TransactionHash`] and [`GasData`] pairs into the store's [`SLED_TX_GAS_DATA_TREE`],
  427. /// committing the changes upon success.
  428. ///
  429. /// This function locks the overlay, verifies that the tx_hashes and gas_data arrays have matching lengths,
  430. /// then inserts them into the store while handling serialization and potential errors. Returns a
  431. /// successful result upon success.
  432. fn insert_tx_gas_data(
  433. &self,
  434. tx_hashes: &[TransactionHash],
  435. gas_data: &[GasData],
  436. lock: &mut MutexGuard<SledDbOverlay>,
  437. ) -> Result<()> {
  438. // Ensure lengths of tx_hashes and gas_data arrays match
  439. if tx_hashes.len() != gas_data.len() {
  440. return Err(Error::Custom(String::from(
  441. "The lengths of tx_hashes and gas_data arrays must match",
  442. )));
  443. }
  444. // Insert each transaction hash and gas data pair
  445. for (tx_hash, gas_data) in tx_hashes.iter().zip(gas_data.iter()) {
  446. // Serialize the gas data
  447. let serialized_gas_data = serialize(gas_data);
  448. // Insert serialized gas data
  449. lock.insert(SLED_TX_GAS_DATA_TREE, tx_hash.inner(), &serialized_gas_data)?;
  450. debug!(target: "explorerd::metrics_store::insert_tx_gas_data", "Inserted gas data for transaction {tx_hash}: {gas_data:?}");
  451. }
  452. Ok(())
  453. }
  454. /// Resets gas metrics in the [`SLED_GAS_METRICS_TREE`] and [`SLED_GAS_METRICS_BY_HEIGHT_TREE`]
  455. /// to a specified block height, undoing all entries after provided height and committing the
  456. /// changes upon success.
  457. ///
  458. /// This function first obtains a lock on the overlay, then reverts changes by calling
  459. /// [`Self::revert_by_height_metrics`] and [`Self::revert_metrics`]. Upon successful revert,
  460. /// all modifications made after the specified height are permanently reverted.
  461. pub fn reset_gas_metrics(&self, height: u32) -> Result<()> {
  462. // Obtain lock
  463. let mut lock = self.overlay.lock().unwrap();
  464. // Revert the metrics by height
  465. self.revert_by_height_metrics(height, &mut lock)?;
  466. // Revert the main metrics entries now that `by_height` tree is reset
  467. self.revert_metrics(&mut lock)?;
  468. // Commit the changes
  469. lock.apply()?;
  470. Ok(())
  471. }
  472. /// Inserts [`GasMetricsKey`] and [`GasMetrics`] pairs into the store's [`SLED_GAS_METRICS_TREE`].
  473. ///
  474. /// This function verifies that the provided keys and metrics arrays have matching lengths,
  475. /// then inserts each pair while handling serialization. Returns a successful result
  476. /// if all insertions are completed without errors.
  477. fn insert(
  478. &self,
  479. keys: &[GasMetricsKey],
  480. metrics: &[GasMetrics],
  481. lock: &mut MutexGuard<SledDbOverlay>,
  482. ) -> Result<()> {
  483. // Ensure lengths of keys and metrics match
  484. if keys.len() != metrics.len() {
  485. return Err(Error::Custom(String::from(
  486. "The lengths of keys and metrics arrays must match",
  487. )));
  488. }
  489. // Insert each metric corresponding to respective gas metrics key
  490. for (key, metric) in keys.iter().zip(metrics.iter()) {
  491. // Insert metric
  492. lock.insert(SLED_GAS_METRICS_TREE, &key.to_sled_key(), &serialize(metric))?;
  493. debug!(target: "explorerd::metrics_store::insert", "Added gas metrics using key {key}: {metric:?}");
  494. }
  495. Ok(())
  496. }
  497. /// Inserts provided [`u32`] height and [`GasMetrics`] pairs into the store's [`SLED_GAS_METRICS_BY_HEIGHT_TREE`].
  498. ///
  499. /// This function verifies matching lengths of provided heights and metrics arrays,
  500. /// and inserts each pair while handling serialization and errors. Returns a successful result
  501. /// if all insertions are completed without errors.
  502. fn insert_by_height(
  503. &self,
  504. heights: &[u32],
  505. metrics: &[GasMetrics],
  506. lock: &mut MutexGuard<SledDbOverlay>,
  507. ) -> Result<()> {
  508. // Ensure lengths of heights and metrics match
  509. if heights.len() != metrics.len() {
  510. return Err(Error::Custom(String::from(
  511. "The lengths of heights and metrics arrays must match",
  512. )));
  513. }
  514. // Insert each metric corresponding to respective height
  515. for (height, metric) in heights.iter().zip(metrics.iter()) {
  516. // Serialize the metric and handle potential errors
  517. let serialized_metric = serialize(metric);
  518. // Insert the serialized metric
  519. lock.insert(
  520. SLED_GAS_METRICS_BY_HEIGHT_TREE,
  521. &height.to_be_bytes(),
  522. &serialized_metric,
  523. )?;
  524. debug!(target: "explorerd::metrics_store::insert_by_height", "Added gas metrics using height {height}: {metric:?}");
  525. }
  526. Ok(())
  527. }
  528. /// This function reverts gas metric entries in the [`SLED_GAS_METRICS_TREE`] to align
  529. /// with the latest metrics state in the [`SLED_GAS_METRICS_BY_HEIGHT_TREE`].
  530. ///
  531. /// It first determines the target timestamp to revert to based on the latest entry
  532. /// in the by_height tree timestamp. Then, it iteratively removes entries from the main metrics
  533. /// tree that are newer than the target timestamp. Once all that is complete, it adds the latest
  534. /// metrics by height to the main metrics tree, returning a successful result if revert processes
  535. /// without error.
  536. fn revert_metrics(&self, lock: &mut MutexGuard<SledDbOverlay>) -> Result<()> {
  537. /*** Determine Metrics To Revert ***/
  538. // Get the last metrics by height and determine the target timestamp to revert to
  539. let latest_by_height = self.get_last_by_height(lock)?;
  540. let target_timestamp = match &latest_by_height {
  541. None => 0,
  542. Some((_, metrics)) => GasMetricsKey::normalize_timestamp(&metrics.timestamp)?.inner(),
  543. };
  544. // Get the timestamp of the latest metrics entry in the metrics store
  545. let mut current_timestamp = match self.get_last(lock)? {
  546. None => return Ok(()),
  547. Some((_, metrics)) => metrics.timestamp.inner(),
  548. };
  549. /*** Revert Main Tree Gas Metrics ***/
  550. // Iterate through at most the total number of gas metric tree entries
  551. for _ in 0..self.db.open_tree(SLED_GAS_METRICS_TREE)?.len() {
  552. // Stop the loop if the current timestamp is less than or equal to the target timestamp,
  553. // as there are no more entries to revert
  554. if current_timestamp <= target_timestamp {
  555. break;
  556. }
  557. // Create a `GasMetricsKey` for the current timestamp to locate the entry to be reverted.
  558. let key_to_revert = GasMetricsKey::new(current_timestamp)?;
  559. // Remove the corresponding entry from the gas metrics tree.
  560. lock.remove(SLED_GAS_METRICS_TREE, &key_to_revert.to_sled_key())?;
  561. info!(target: "explorerd:metrics_store:revert_metrics", "Successfully reverted metrics with key: {key_to_revert}");
  562. // Move to the previous valid timestamp by subtracting the defined time interval
  563. current_timestamp = current_timestamp.saturating_sub(GAS_METRICS_KEY_TIME_INTERVAL);
  564. }
  565. /*** Add the Latest Reverted Metrics To Main Tree ***/
  566. // Retrieve the latest metrics from the `by_height` tree and normalize its timestamp so it can be added to the main tree.
  567. // If there are no metrics in the `by_height` tree, we may have reset to 0, so return as there is nothing add.
  568. let latest_metrics = match latest_by_height {
  569. None => return Ok(()),
  570. Some((_, mut metrics)) => {
  571. metrics.timestamp = GasMetricsKey::normalize_timestamp(&metrics.timestamp)?;
  572. metrics
  573. }
  574. };
  575. // Add the latest metrics to the main tree based on latest reverted metrics by height
  576. let gas_metrics_key = GasMetricsKey::new(&latest_metrics.timestamp)?;
  577. self.insert(&[gas_metrics_key], &[latest_metrics], lock)?;
  578. Ok(())
  579. }
  580. /// Reverts gas metric entries from [`SLED_GAS_METRICS_BY_HEIGHT_TREE`] to provided `height`.
  581. ///
  582. /// This function iterates through the entries in gas metrics by height tree and removes all entries
  583. /// with heights greater than the specified `height`, effectively reverting all gas metrics beyond that point.
  584. fn revert_by_height_metrics(
  585. &self,
  586. height: u32,
  587. lock: &mut MutexGuard<SledDbOverlay>,
  588. ) -> Result<()> {
  589. // Retrieve the last stored block height
  590. let (last_height, _) = match self.get_last_by_height(lock)? {
  591. None => return Ok(()),
  592. Some(v) => v,
  593. };
  594. // Return early if the requested height is after the last stored height
  595. if height >= last_height {
  596. return Ok(());
  597. }
  598. // Remove keys greater than `height`
  599. while let Some((cur_height_bytes, _)) = lock.last(SLED_GAS_METRICS_BY_HEIGHT_TREE)? {
  600. // Convert height bytes to u32
  601. let cur_height = u32::from_be_bytes(cur_height_bytes.as_ref().try_into()?);
  602. // Process all heights that are bigger than provided `height`
  603. if cur_height <= height {
  604. break;
  605. }
  606. // Remove height being reverted
  607. lock.remove(SLED_GAS_METRICS_BY_HEIGHT_TREE, &cur_height_bytes)?;
  608. info!(target: "explorerd:metrics_store:revert_by_height_metrics", "Successfully reverted metrics with height: {cur_height}");
  609. }
  610. Ok(())
  611. }
  612. /// Fetches the most recent gas metrics from [`SLED_GAS_METRICS_TREE`], returning an option
  613. /// containing a metrics key [`GasMetricsKey`] and [`GasMetrics`] pair, or `None` if no metrics exist.
  614. fn get_last(
  615. &self,
  616. lock: &mut MutexGuard<SledDbOverlay>,
  617. ) -> Result<Option<(GasMetricsKey, GasMetrics)>> {
  618. // Fetch and deserialize key and metric pair
  619. lock.last(SLED_GAS_METRICS_TREE)?
  620. .map(|(key_bytes, metrics_bytes)| {
  621. // Deserialize the metrics key
  622. let key = GasMetricsKey::from_sled_key(&key_bytes)?;
  623. // Deserialize the stored gas metrics
  624. let metrics: GasMetrics = deserialize(&metrics_bytes).map_err(Error::from)?;
  625. Ok((key, metrics))
  626. })
  627. .transpose()
  628. }
  629. /// Fetches the most recent gas metrics from [`SLED_GAS_METRICS_BY_HEIGHT_TREE`], returning an option
  630. /// containing a height [`u32`] and [`GasMetrics`] pair, or `None` if no metrics exist.
  631. fn get_last_by_height(
  632. &self,
  633. lock: &mut MutexGuard<SledDbOverlay>,
  634. ) -> Result<Option<(u32, GasMetrics)>> {
  635. // Fetch and deserialize height and metric pair
  636. lock.last(SLED_GAS_METRICS_BY_HEIGHT_TREE)?
  637. .map(|(height_bytes, metrics_bytes)| {
  638. // Deserialize the height
  639. let key_bytes: [u8; 4] = height_bytes.as_ref().try_into().unwrap();
  640. let height = u32::from_be_bytes(key_bytes);
  641. // Deserialize the stored gas metrics
  642. let metrics: GasMetrics = deserialize(&metrics_bytes).map_err(Error::from)?;
  643. Ok((height, metrics))
  644. })
  645. .transpose()
  646. }
  647. }
  648. /// Represents a key used to store and fetch metrics in the metrics store.
  649. ///
  650. /// This struct provides methods for creating, serializing, and deserializing gas metrics keys.
  651. /// It supports creation from various time representations through the [`GasMetricsKeySource`] trait
  652. /// and offers conversion methods for use with a sled database.
  653. #[derive(Debug, Eq, PartialEq, Clone)]
  654. pub struct GasMetricsKey(pub DateTime);
  655. impl GasMetricsKey {
  656. /// Creates a new [`GasMetricsKey`] from a source that implements [`GasMetricsKeySource`].
  657. /// Depending on the use case, the key supports different input sources such as `Timestamp`, `u64` timestamp,
  658. /// or `&str` timestamp to create the key.
  659. pub fn new<T: GasMetricsKeySource>(source: T) -> Result<GasMetricsKey> {
  660. source.to_key()
  661. }
  662. /// Gets the inner [`DateTime`] value.
  663. pub fn inner(&self) -> &DateTime {
  664. &self.0
  665. }
  666. /// Converts the [`GasMetricsKey`] into a key suitable for use with a sled database.
  667. pub fn to_sled_key(&self) -> Vec<u8> {
  668. // Create a new vector with a capacity of 28 bytes
  669. let mut sled_key = Vec::with_capacity(28);
  670. // Push the byte representations of each field into the vector
  671. sled_key.extend_from_slice(&self.inner().year.to_be_bytes());
  672. sled_key.extend_from_slice(&self.inner().month.to_be_bytes());
  673. sled_key.extend_from_slice(&self.inner().day.to_be_bytes());
  674. sled_key.extend_from_slice(&self.inner().hour.to_be_bytes());
  675. sled_key.extend_from_slice(&self.inner().min.to_be_bytes());
  676. sled_key.extend_from_slice(&self.inner().sec.to_be_bytes());
  677. sled_key.extend_from_slice(&self.inner().nanos.to_be_bytes());
  678. // Return sled key
  679. sled_key
  680. }
  681. /// Converts a `sled` key into a [`GasMetricsKey`] by deserializing a slice of bytes.
  682. pub fn from_sled_key(bytes: &[u8]) -> Result<Self> {
  683. if bytes.len() != 28 {
  684. return Err(Error::Custom(String::from("Invalid byte length for GasMetricsKey")));
  685. }
  686. // Deserialize byte representations into each field
  687. let key = DateTime {
  688. year: u32::from_be_bytes(bytes[0..4].try_into()?),
  689. month: u32::from_be_bytes(bytes[4..8].try_into()?),
  690. day: u32::from_be_bytes(bytes[8..12].try_into()?),
  691. hour: u32::from_be_bytes(bytes[12..16].try_into()?),
  692. min: u32::from_be_bytes(bytes[16..20].try_into()?),
  693. sec: u32::from_be_bytes(bytes[20..24].try_into()?),
  694. nanos: u32::from_be_bytes(bytes[24..28].try_into()?),
  695. };
  696. Ok(Self(key))
  697. }
  698. /// Normalizes the given [`DateTime`] to the start of hour.
  699. pub fn normalize_date_time(date_time: DateTime) -> DateTime {
  700. DateTime {
  701. nanos: 0,
  702. sec: 0,
  703. min: 0,
  704. hour: date_time.hour,
  705. day: date_time.day,
  706. month: date_time.month,
  707. year: date_time.year,
  708. }
  709. }
  710. /// Normalizes a given [`Timestamp`] to the start of the hour.
  711. pub fn normalize_timestamp(timestamp: &Timestamp) -> Result<Timestamp> {
  712. let remainder = timestamp.inner() % GAS_METRICS_KEY_TIME_INTERVAL;
  713. timestamp.checked_sub(Timestamp::from_u64(remainder))
  714. }
  715. }
  716. impl fmt::Display for GasMetricsKey {
  717. fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
  718. write!(f, "{}", self.inner())
  719. }
  720. }
  721. /// Provides a unified method for creating new instances of GasMetricKeys using
  722. /// various time representations: [`Timestamp`], `u64` timestamp, or `&str` timestamp.
  723. pub trait GasMetricsKeySource {
  724. fn to_key(&self) -> Result<GasMetricsKey>;
  725. }
  726. /// Implements [`GasMetricsKeySource`] for &[`Timestamp`], converting it to a [`GasMetricsKey`].
  727. impl GasMetricsKeySource for &Timestamp {
  728. fn to_key(&self) -> Result<GasMetricsKey> {
  729. let date_time = DateTime::from_timestamp(self.inner(), 0);
  730. Ok(GasMetricsKey(GasMetricsKey::normalize_date_time(date_time)))
  731. }
  732. }
  733. /// Implements [`GasMetricsKeySource`] for `u64`, converting it to a [`GasMetricsKey`].
  734. impl GasMetricsKeySource for u64 {
  735. fn to_key(&self) -> Result<GasMetricsKey> {
  736. let date_time = DateTime::from_timestamp(*self, 0);
  737. Ok(GasMetricsKey(GasMetricsKey::normalize_date_time(date_time)))
  738. }
  739. }
  740. /// Implements [`GasMetricsKeySource`] for string slices, converting a `&str` in the `YYYY-MM-DD HH:mm:ss UTC` format
  741. /// to a [`GasMetricsKey`]. Returns an [`Error::ParseFailed`] error if the provided timestamp string slice is invalid.
  742. impl GasMetricsKeySource for &str {
  743. fn to_key(&self) -> Result<GasMetricsKey> {
  744. let date_time = DateTime::from_timestamp_str(self)?;
  745. Ok(GasMetricsKey(GasMetricsKey::normalize_date_time(date_time)))
  746. }
  747. }
  748. #[cfg(test)]
  749. /// This test module verifies the correct insertion, retrieval, and reset of metrics in the store.
  750. /// It covers adding metrics, searching metrics by time and transaction hash, and resetting metrics with specified heights.
  751. mod tests {
  752. use std::{
  753. str::FromStr,
  754. time::{Duration, SystemTime, UNIX_EPOCH},
  755. };
  756. use darkfi::util::{
  757. logger::{setup_test_logger, Level},
  758. time::DateTime,
  759. };
  760. use structopt::lazy_static::lazy_static;
  761. use tracing::warn;
  762. use super::*;
  763. /// Number of heights to simulate.
  764. const HEIGHT: u32 = 10;
  765. /// Fixed timestamp in seconds since UNIX epoch.
  766. const FIXED_TIMESTAMP: u64 = 1732042800;
  767. /// [`FIXED_TIMESTAMP`] timestamp as a string in UTC format.
  768. const FIXED_TIMESTAMP_STR: &str = "2024-11-19T19:00:00";
  769. lazy_static! {
  770. /// Test transaction hash.
  771. pub static ref TX_HASH: TransactionHash = TransactionHash::from_str(
  772. "92225ff00a3755d8df93c626b59f6e36cf021d85ebccecdedc38f3f1890a15fc"
  773. ).expect("Invalid transaction hash");
  774. }
  775. /// Tests inserting gas metrics, verifying the correctness of stored metrics.
  776. #[test]
  777. fn test_insert_gas_metrics() -> Result<()> {
  778. // Declare constants used for test
  779. const EXPECTED_HEIGHT: usize = HEIGHT as usize - 1;
  780. // Setup test, returning initialized metrics store
  781. let store = setup()?;
  782. // Load test data into the store and get the expected metrics results
  783. let test_data = load_random_metrics(&store, |_, _| {})?;
  784. // Verify metrics were inserted with the expected counts
  785. assert_eq!(store.len(), EXPECTED_HEIGHT);
  786. // Process height 0 test data separately
  787. let mut test_data_iter = test_data.iter();
  788. // For height 0, confirm there are no metrics stored in the store
  789. if let Some(test_data_height0) = test_data_iter.next() {
  790. let actual_height0 = store.get(&[GasMetricsKey::new(&test_data_height0.timestamp)?])?;
  791. assert!(
  792. actual_height0.is_empty(),
  793. "Timestamp associated with height 0 should not have any metrics stored"
  794. );
  795. }
  796. // Process remaining test data, verifying that each stored metric matches expected results
  797. for expected in test_data_iter {
  798. let actual = store.get(&[GasMetricsKey::new(&expected.timestamp)?])?;
  799. let expected_normalized = normalize_metrics_timestamp(expected)?;
  800. assert_eq!(&expected_normalized, &actual[0]);
  801. }
  802. Ok(())
  803. }
  804. /// Tests inserting gas metrics into the `by_height` tree, verifying the correctness of stored metrics.
  805. #[test]
  806. fn test_insert_by_height_gas_metrics() -> Result<()> {
  807. // Declare constants used for test
  808. const EXPECTED_HEIGHT: usize = HEIGHT as usize - 1;
  809. // Setup test, returning initialized metrics store
  810. let store = setup()?;
  811. // Load test data into the store and get the expected metrics results
  812. let test_data = load_random_metrics(&store, |_, _| {})?;
  813. // Verify metrics were inserted with the expected counts
  814. assert_eq!(store.len(), EXPECTED_HEIGHT);
  815. // For height 0, confirm there are no metrics stored in metrics by height
  816. let actual_height0 = store.get_by_height(&[0])?;
  817. assert!(actual_height0.is_empty(), "Height 0 should not have any metrics stored");
  818. // Process remaining heights, verifying that each stored metric matches expected results
  819. for (height, expected) in (1..).zip(test_data.iter().skip(1)) {
  820. let actual = store.get_by_height(&[height])?;
  821. assert!(!actual.is_empty(), "No metrics found for height {height}");
  822. assert_eq!(expected, &actual[0]);
  823. }
  824. Ok(())
  825. }
  826. /// Tests searching gas metrics by the hour, verifying the correct metrics are found
  827. /// and match expected values.
  828. #[test]
  829. fn test_search_metrics_by_hour() -> Result<()> {
  830. // Setup test, returning initialized metrics store
  831. let store = setup()?;
  832. // Load test data, initializing expected with the fourth loaded record
  833. let expected = &load_random_metrics(&store, |_, _| {})?[3];
  834. // Create search criteria based on the expected timestamp value
  835. let search_criteria = DateTime::from_timestamp(expected.timestamp.inner(), 0);
  836. // Search metrics by the hour
  837. let actual_opt = store.main.iter().find_map(|res| {
  838. res.ok().and_then(|(k, v)| {
  839. let key = GasMetricsKey::from_sled_key(&k).ok()?;
  840. if key.inner().hour == search_criteria.hour {
  841. deserialize::<GasMetrics>(&v).ok()
  842. } else {
  843. None
  844. }
  845. })
  846. });
  847. // Verify the found metrics match expected results
  848. assert!(actual_opt.is_some());
  849. assert_eq!(normalize_metrics_timestamp(expected)?, actual_opt.unwrap());
  850. Ok(())
  851. }
  852. /// Tests fetching gas metrics by a timestamp string, verifying the retrieved metrics
  853. /// match expected values.
  854. #[test]
  855. fn test_get_metrics_by_timestamp_str() -> Result<()> {
  856. // Setup test, returning initialized metrics store
  857. let store = setup()?;
  858. // Load fixed data needed for test, initializing expected with the first loaded record
  859. let (expected, _) = &load_fixed_metrics(&store)?[0];
  860. // Create gas metrics key using a test fixed timestamp
  861. let gas_metrics_key = GasMetricsKey::new(FIXED_TIMESTAMP_STR)?;
  862. // Verify the key retrieves the correct metrics and matches the expected value
  863. let actual = store.get(&[gas_metrics_key])?;
  864. assert_eq!(expected, &actual[0]);
  865. Ok(())
  866. }
  867. /// Tests the insertion and retrieval of transaction gas data in the store, verifying expected results.
  868. /// Additionally, it tests that transactions not found in the store correctly return a `None` result.
  869. #[test]
  870. fn test_tx_gas_data() -> Result<()> {
  871. let tx_hash_not_found: TransactionHash = TransactionHash::from_str(
  872. "93325ff00a3755d8df93c626b59f6e36cf021d85ebccecdedc38f3f1890a15fc",
  873. )
  874. .expect("Invalid hash");
  875. // Setup test, returning initialized metrics store
  876. let store = setup()?;
  877. // Load data needed for test, initializing expected with the first loaded record
  878. let (_, expected) = &load_fixed_metrics(&store)?[0];
  879. // Verify that existing transaction is found
  880. let actual_opt = store.get_tx_gas_data(&TX_HASH)?;
  881. assert!(actual_opt.is_some());
  882. assert_eq!(*expected, actual_opt.unwrap());
  883. // Verify that transactions that do not exist return None result
  884. let actual_not_found = store.get_tx_gas_data(&tx_hash_not_found)?;
  885. assert_eq!(None, actual_not_found);
  886. Ok(())
  887. }
  888. /// Tests resetting gas metrics within a specified height range, verifying that both the `by_height` and `main` trees
  889. /// are properly set to the reset height.
  890. #[test]
  891. fn test_reset_metrics_within_height_range() -> Result<()> {
  892. // Declare constants used for test
  893. const RESET_HEIGHT: u32 = 6;
  894. // Setup test, returning initialized metrics store
  895. let store = setup()?;
  896. // Load test data into the store and get the expected reset metrics result
  897. let expected = load_reset_metrics(&store, RESET_HEIGHT)?;
  898. // Reset metrics
  899. store.reset_gas_metrics(RESET_HEIGHT)?;
  900. // Fetch reset metrics by height
  901. let actual_by_height_opt = store.get_last_by_height()?;
  902. assert!(actual_by_height_opt.is_some(), "Expected get_last_by_height to return metrics");
  903. // Verify metrics by height are properly reset
  904. let (_, actual_by_height) = actual_by_height_opt.unwrap();
  905. assert_eq!(&expected, &actual_by_height);
  906. // Fetch reset main metrics
  907. let actual_main_opt = store.get_last()?;
  908. assert!(actual_main_opt.is_some(), "Expected get_last to return metrics");
  909. // Verify main metrics are properly reset
  910. let (_, actual_main_metrics) = actual_main_opt.unwrap();
  911. assert_eq!(&normalize_metrics_timestamp(&expected)?, &actual_main_metrics);
  912. Ok(())
  913. }
  914. /// Tests resetting the metrics store to height 0, ensuring it handles the operation gracefully without errors
  915. /// and verifies that no metrics remain in the store afterward.
  916. #[test]
  917. fn test_reset_metrics_height_to_0() -> Result<()> {
  918. // Declare constants used for test
  919. const RESET_HEIGHT: u32 = 0;
  920. const EXPECTED_RESET_HEIGHT: usize = 0;
  921. // Setup test, returning initialized metrics store
  922. let store = setup()?;
  923. // Load reset test data needed for test
  924. _ = load_reset_metrics(&store, RESET_HEIGHT)?;
  925. // Reset metrics
  926. store.reset_gas_metrics(RESET_HEIGHT)?;
  927. // Verify metrics were reset with the expected counts
  928. assert_eq!(store.len_by_height(), EXPECTED_RESET_HEIGHT);
  929. assert_eq!(store.len(), EXPECTED_RESET_HEIGHT);
  930. // Verify metrics by height are empty
  931. let actual_by_height_opt = store.get_last_by_height()?;
  932. assert!(actual_by_height_opt.is_none(), "Expected None from get_last_by_height");
  933. // Confirm main metrics are empty
  934. let actual_main_opt = store.get_last()?;
  935. assert!(actual_main_opt.is_none(), "Expected None from get_last");
  936. Ok(())
  937. }
  938. /// Tests that resetting beyond the number of available metrics does not change
  939. /// the store and no errors are thrown since there are no metrics to reset.
  940. #[test]
  941. fn test_reset_metrics_beyond_height() -> Result<()> {
  942. // Declare constants used for test
  943. const RESET_HEIGHT: u32 = HEIGHT + 1;
  944. const EXPECTED_RESET_HEIGHT: usize = HEIGHT as usize - 1;
  945. // Setup test, returning initialized metrics store
  946. let store = setup()?;
  947. // Load reset test data needed for test, storing the expected result
  948. let expected = load_reset_metrics(&store, RESET_HEIGHT)?;
  949. // Reset metrics to given height
  950. store.reset_gas_metrics(RESET_HEIGHT)?;
  951. // Verify metrics were reset with the expected counts
  952. assert_eq!(store.len_by_height(), EXPECTED_RESET_HEIGHT);
  953. assert_eq!(store.len(), EXPECTED_RESET_HEIGHT);
  954. // Verify that the last record for metrics by height is correctly reset
  955. let actual_by_height_opt = store.get_last_by_height()?;
  956. assert!(actual_by_height_opt.is_some(), "Expected get_last_by_height to return metrics");
  957. let (_, actual_by_height) = actual_by_height_opt.unwrap();
  958. assert_eq!(&expected, &actual_by_height);
  959. // Verify that the last record for main metrics is correctly reset
  960. let actual_main_opt = store.get_last()?;
  961. assert!(actual_main_opt.is_some(), "Expected get_last to return metrics");
  962. let (_, actual_main) = actual_main_opt.unwrap();
  963. assert_eq!(&normalize_metrics_timestamp(&expected)?, &actual_main);
  964. Ok(())
  965. }
  966. /// Tests resetting metrics at the last available height to verify that the code
  967. /// can handle the boundary condition.
  968. #[test]
  969. fn test_reset_metrics_at_height() -> Result<()> {
  970. // Declare constants used for test
  971. const RESET_HEIGHT: u32 = HEIGHT;
  972. const EXPECTED_RESET_HEIGHT: usize = HEIGHT as usize - 1;
  973. // Setup test, returning initialized metrics store
  974. let store = setup()?;
  975. // Load reset test data needed for test
  976. let expected = load_reset_metrics(&store, RESET_HEIGHT)?;
  977. // Reset metrics to given height
  978. store.reset_gas_metrics(RESET_HEIGHT)?;
  979. // Verify metrics were reset with the expected counts
  980. assert_eq!(store.len_by_height(), EXPECTED_RESET_HEIGHT);
  981. assert_eq!(store.len(), EXPECTED_RESET_HEIGHT);
  982. // Verify that the last record for metrics by height is correctly reset
  983. let actual_by_height_opt = store.get_last_by_height()?;
  984. assert!(actual_by_height_opt.is_some(), "Expected get_last_by_height to return metrics");
  985. let (_, actual_by_height) = actual_by_height_opt.unwrap();
  986. assert_eq!(&expected, &actual_by_height);
  987. // Verify that the last record for main metrics is correctly reset
  988. let actual_main_opt = store.get_last()?;
  989. assert!(actual_main_opt.is_some(), "Expected get_last to return metrics");
  990. let (_, actual_main) = actual_main_opt.unwrap();
  991. assert_eq!(&normalize_metrics_timestamp(&expected)?, &actual_main);
  992. Ok(())
  993. }
  994. /// Tests that resetting an empty metrics store gracefully handles
  995. /// the operation without errors and ensures the store remains empty.
  996. #[test]
  997. fn test_reset_empty_store() -> Result<()> {
  998. const RESET_HEIGHT: u32 = 6;
  999. // Setup test, returning initialized metrics store
  1000. let store = setup()?;
  1001. // Reset metrics with an empty store
  1002. store.reset_gas_metrics(RESET_HEIGHT)?;
  1003. // Verify no metrics with the expected counts
  1004. assert_eq!(store.len_by_height(), 0);
  1005. assert_eq!(store.len(), 0);
  1006. // Verify that metrics by height is empty
  1007. let actual_by_height = store.get_last_by_height()?;
  1008. assert!(actual_by_height.is_none(), "Expected get_last_by_height to return None");
  1009. // Verify main metrics is empty
  1010. let actual_main = store.get_last()?;
  1011. assert!(actual_main.is_none(), "Expected get_last to return None");
  1012. Ok(())
  1013. }
  1014. /// Sets up a test case for metrics store testing by initializing the logger,
  1015. /// creating a temporary database, and returning an initialized metrics store.
  1016. fn setup() -> Result<MetricsStore> {
  1017. // Initialize logger to show execution output
  1018. if setup_test_logger(
  1019. &["sled", "runtime", "net"],
  1020. false,
  1021. Level::Info,
  1022. //Level::Verbose,
  1023. //Level::Debug,
  1024. //Level::Trace,
  1025. )
  1026. .is_err()
  1027. {
  1028. warn!("Logger already initialized");
  1029. }
  1030. // Create a temporary directory for the sled database
  1031. let db =
  1032. sled::Config::new().temporary(true).open().expect("Unable to open test sled database");
  1033. // Initialize the metrics store
  1034. let metrics_store = MetricsStore::new(&db.clone())?;
  1035. Ok(metrics_store)
  1036. }
  1037. /// Loads random test gas metrics data into the given metrics store, simulating height 0 as a
  1038. /// genesis block with no metrics.
  1039. ///
  1040. /// Computes the starting block timestamp from the current system time for the first metric,
  1041. /// then inserts each subsequent metric at intervals of [`GAS_METRICS_KEY_TIME_INTERVAL`],
  1042. /// resulting in metrics being inserted one hour apart. The function iterates through a predefined
  1043. /// height range, as defined by [`HEIGHT`], to accumulate and insert gas metrics. After each
  1044. /// metric is stored, the `metric_loaded` closure is invoked, allowing the caller to perform
  1045. /// specific actions as the data is loaded.
  1046. ///
  1047. /// NOTE: A fixed transaction hash is used to insert the metrics, as this test data is solely intended
  1048. /// to validate gas metrics and not transaction-specific gas data.
  1049. ///
  1050. /// Upon success, it returns a list of snapshots of the accumulated metrics that were loaded.
  1051. fn load_random_metrics<F>(
  1052. metrics_store: &MetricsStore,
  1053. mut metrics_loaded: F,
  1054. ) -> Result<Vec<GasMetrics>>
  1055. where
  1056. F: FnMut(u32, &GasMetrics),
  1057. {
  1058. // Calculate the start block timestamp
  1059. let start_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
  1060. // Define variables to store accumulated loaded data
  1061. let mut accumulated_metrics = GasMetrics::default();
  1062. let mut metrics = Vec::with_capacity(HEIGHT as usize);
  1063. // Iterate and load data
  1064. for height in 0..HEIGHT {
  1065. let timestamp_secs = (UNIX_EPOCH +
  1066. Duration::from_secs(start_time + height as u64 * GAS_METRICS_KEY_TIME_INTERVAL))
  1067. .duration_since(UNIX_EPOCH)?
  1068. .as_secs();
  1069. // Initialize simulated block_timestamp
  1070. let block_timestamp = Timestamp::from(timestamp_secs);
  1071. accumulated_metrics.timestamp = block_timestamp;
  1072. // Simulate genesis block, metrics are stored after height 0
  1073. if height > 0 {
  1074. let tx_gas_data = random_gas_data(height as u64 + start_time);
  1075. accumulated_metrics.add(slice::from_ref(&tx_gas_data));
  1076. metrics_store.insert_gas_metrics(
  1077. height,
  1078. &block_timestamp,
  1079. &[*TX_HASH],
  1080. &[tx_gas_data],
  1081. )?;
  1082. }
  1083. // Invoke passed in metrics loaded closure
  1084. metrics_loaded(height, &accumulated_metrics);
  1085. // Add a snapshot of the accumulated metrics
  1086. metrics.push(accumulated_metrics.clone());
  1087. }
  1088. Ok(metrics)
  1089. }
  1090. /// Loads fixed test data into the metrics store using fixed timestamps,
  1091. /// returning snapshots of accumulated [`GasMetrics`] with corresponding [`GasData`]
  1092. /// used to update the metrics.
  1093. ///
  1094. /// Currently, this function only loads a single record but is designed to be extendable
  1095. /// to insert additional records in the future without affecting the method's return signature,
  1096. /// making it suitable for use in tests.
  1097. fn load_fixed_metrics(metrics_store: &MetricsStore) -> Result<Vec<(GasMetrics, GasData)>> {
  1098. // Convert the fixed timestamp constant to a Timestamp object
  1099. let fixed_timestamp = Timestamp::from_u64(FIXED_TIMESTAMP);
  1100. // Initialize an empty GasMetrics object to accumulate the data
  1101. let height: u32 = 1;
  1102. let mut accumulated_metrics = GasMetrics::default();
  1103. let mut metrics_vec = Vec::with_capacity(HEIGHT as usize);
  1104. // Initialize the block_timestamp using the fixed timestamp
  1105. let block_timestamp = fixed_timestamp;
  1106. accumulated_metrics.timestamp = block_timestamp;
  1107. // Generate random gas data for the given height
  1108. let gas_data = random_gas_data(height as u64);
  1109. accumulated_metrics.add(slice::from_ref(&gas_data));
  1110. // Insert the gas metrics into the metrics store
  1111. metrics_store.insert_gas_metrics(
  1112. height,
  1113. &block_timestamp,
  1114. &[*TX_HASH],
  1115. slice::from_ref(&gas_data),
  1116. )?;
  1117. metrics_vec.push((accumulated_metrics, gas_data));
  1118. Ok(metrics_vec)
  1119. }
  1120. /// Loads reset test data into the store, returning the accumulated gas metrics at the specified reset height.
  1121. fn load_reset_metrics(metrics_store: &MetricsStore, reset_height: u32) -> Result<GasMetrics> {
  1122. let mut reset_metrics = GasMetrics::default();
  1123. // Load metrics, passing in a closure to store the reset metrics
  1124. _ = load_random_metrics(metrics_store, |height, acc_metrics| {
  1125. // Store accumulated metrics at reset height
  1126. if reset_height == height || reset_height >= HEIGHT {
  1127. reset_metrics = acc_metrics.clone();
  1128. }
  1129. })?;
  1130. Ok(reset_metrics)
  1131. }
  1132. /// Generates random [`GasData`] based on the provided seed value, allowing for the simulation
  1133. /// of varied gas data values.
  1134. fn random_gas_data(seed: u64) -> GasData {
  1135. /// Defines a limit for gas data values.
  1136. const GAS_LIMIT: u64 = 100_000;
  1137. // Initialize gas usage with the provided seed
  1138. let mut gas_used = seed;
  1139. // Closure to generate a random gas value
  1140. let mut random_gas = || {
  1141. // Introduce variability using the seed and current gas_used
  1142. let variation = seed.wrapping_add(gas_used);
  1143. gas_used = gas_used.wrapping_mul(6364136223846793005).wrapping_add(variation);
  1144. gas_used
  1145. };
  1146. // Create GasData with random values constrained by GAS_LIMIT
  1147. GasData {
  1148. paid: random_gas() % GAS_LIMIT,
  1149. wasm: random_gas() % GAS_LIMIT,
  1150. zk_circuits: random_gas() % GAS_LIMIT,
  1151. signatures: random_gas() % GAS_LIMIT,
  1152. deployments: random_gas() % GAS_LIMIT,
  1153. }
  1154. }
  1155. /// Normalizes the [`GasMetrics`] timestamp to the start of the hour for test comparisons.
  1156. fn normalize_metrics_timestamp(metrics: &GasMetrics) -> Result<GasMetrics> {
  1157. let mut normalized_metrics = metrics.clone();
  1158. normalized_metrics.timestamp = GasMetricsKey::normalize_timestamp(&metrics.timestamp)?;
  1159. Ok(normalized_metrics)
  1160. }
  1161. }