metrics.rs 57 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376
  1. /* This file is part of DarkFi (https://dark.fi)
  2. *
  3. * Copyright (C) 2020-2025 Dyne.org foundation
  4. *
  5. * This program is free software: you can redistribute it and/or modify
  6. * it under the terms of the GNU Affero General Public License as
  7. * published by the Free Software Foundation, either version 3 of the
  8. * License, or (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. * GNU Affero General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU Affero General Public License
  16. * along with this program. If not, see <https://www.gnu.org/licenses/>.
  17. */
  18. use std::{
  19. fmt, slice,
  20. sync::{Arc, Mutex, MutexGuard},
  21. };
  22. use sled_overlay::{sled, SledDbOverlay};
  23. use tracing::{debug, info};
  24. use darkfi::{
  25. blockchain::SledDbOverlayPtr,
  26. util::time::{DateTime, Timestamp},
  27. validator::fees::GasData,
  28. Error, Result,
  29. };
  30. use darkfi_sdk::{num_traits::ToBytes, tx::TransactionHash};
  31. use darkfi_serial::{async_trait, deserialize, serialize, SerialDecodable, SerialEncodable};
  32. /// Gas metrics tree name.
  33. pub const SLED_GAS_METRICS_TREE: &[u8] = b"_gas_metrics";
  34. /// Gas metrics `by_height` tree that contains all metrics by height.
  35. pub const SLED_GAS_METRICS_BY_HEIGHT_TREE: &[u8] = b"_gas_metrics_by_height";
  36. /// Transaction gas data tree name.
  37. pub const SLED_TX_GAS_DATA_TREE: &[u8] = b"_tx_gas_data";
  38. /// The time interval for [`GasMetricsKey`]s in the main tree, specified in seconds.
  39. /// Metrics are stored in hourly intervals (3600 seconds), meaning all metrics accumulated
  40. /// within a specific hour are stored using a key representing the start of that hour.
  41. pub const GAS_METRICS_KEY_TIME_INTERVAL: u64 = 3600;
  42. #[derive(Debug, Clone, Default, Eq, PartialEq, SerialEncodable, SerialDecodable)]
  43. /// Represents metrics used to capture key statistical data.
  44. pub struct Metrics {
  45. /// An aggregate value that represents the sum of the metrics.
  46. pub sum: u64,
  47. /// The smallest value in the series of measured metrics.
  48. pub min: u64,
  49. /// The largest value in the series of measured metrics.
  50. pub max: u64,
  51. }
  52. // Temporarily disable unused warnings until the store is integrated with the explorer
  53. #[allow(dead_code)]
  54. impl Metrics {
  55. /// Constructs a [`Metrics`] instance with provided parameters.
  56. pub fn new(sum: u64, min: u64, max: u64) -> Self {
  57. Self { sum, min, max }
  58. }
  59. }
  60. /// Structure for managing gas metrics across all transactions in the store.
  61. ///
  62. /// This struct maintains running totals, extrema, and transaction counts to efficiently calculate
  63. /// metrics without the need to iterate through previous transactions when new data is added. It is used to build a
  64. /// comprehensive view of gas metrics across the blockchain's history, including total gas, WASM gas,
  65. /// ZK circuit gas, and signature gas. The structure allows for O(1) performance in calculating
  66. /// averages and updating min/max values.
  67. #[derive(Clone, Default, Eq, PartialEq, SerialEncodable, SerialDecodable)]
  68. pub struct GasMetrics {
  69. /// Represents the total count of transactions tracked by the metrics store.
  70. pub txs_count: u64,
  71. /// Overall gas consumed metrics across all transactions.
  72. pub total_gas: Metrics,
  73. /// Gas used across all executed wasm transactions.
  74. pub wasm_gas: Metrics,
  75. /// Gas consumed across all zk circuit computations.
  76. pub zk_circuits_gas: Metrics,
  77. /// Gas used metrics related to signatures across transactions.
  78. pub signatures_gas: Metrics,
  79. /// Gas consumed for deployments across transactions.
  80. pub deployments_gas: Metrics,
  81. /// The time the metrics was calculated
  82. pub timestamp: Timestamp,
  83. }
  84. // Temporarily disable unused warnings until the store is integrated with the explorer
  85. #[allow(dead_code)]
  86. impl GasMetrics {
  87. /// Creates a [`GasMetrics`] instance.
  88. pub fn new(
  89. txs_count: u64,
  90. total_gas: Metrics,
  91. wasm_gas: Metrics,
  92. zk_circuit_gas: Metrics,
  93. signature_gas: Metrics,
  94. deployment_gas: Metrics,
  95. timestamp: Timestamp,
  96. ) -> Self {
  97. Self {
  98. txs_count,
  99. total_gas,
  100. wasm_gas,
  101. zk_circuits_gas: zk_circuit_gas,
  102. signatures_gas: signature_gas,
  103. deployments_gas: deployment_gas,
  104. timestamp,
  105. }
  106. }
  107. /// Provides the average of the total gas used.
  108. pub fn avg_total_gas_used(&self) -> u64 {
  109. self.total_gas.sum.checked_div(self.txs_count).unwrap_or_default()
  110. }
  111. /// Provides the average of the gas used across WASM transactions.
  112. pub fn avg_wasm_gas_used(&self) -> u64 {
  113. self.wasm_gas.sum.checked_div(self.txs_count).unwrap_or_default()
  114. }
  115. /// Provides the average of the gas consumed across Zero-Knowledge Circuit computations.
  116. pub fn avg_zk_circuits_gas_used(&self) -> u64 {
  117. self.zk_circuits_gas.sum.checked_div(self.txs_count).unwrap_or_default()
  118. }
  119. /// Provides the average of the gas used to sign transactions.
  120. pub fn avg_signatures_gas_used(&self) -> u64 {
  121. self.signatures_gas.sum.checked_div(self.txs_count).unwrap_or_default()
  122. }
  123. /// Provides the average of the gas used for deployments.
  124. pub fn avg_deployments_gas_used(&self) -> u64 {
  125. self.deployments_gas.sum.checked_div(self.txs_count).unwrap_or_default()
  126. }
  127. /// Adds new [`GasData`] to the existing accumulated values.
  128. ///
  129. /// This method updates running totals, transaction counts, and min/max values
  130. /// for various gas metric categories. It accumulates new data without reading existing
  131. /// averages, minimums, or maximums from the database to optimize performance.
  132. pub fn add(&mut self, tx_gas_data: &[GasData]) {
  133. for gas_data in tx_gas_data {
  134. // Increment number of transactions included in stats
  135. self.txs_count += 1;
  136. // Update the statistics related to total gas
  137. self.total_gas.sum += gas_data.total_gas_used();
  138. // Update the statistics related to WASM gas
  139. self.wasm_gas.sum += gas_data.wasm;
  140. // Update the statistics related to ZK circuit gas
  141. self.zk_circuits_gas.sum += gas_data.zk_circuits;
  142. // Update the statistics related to signature gas
  143. self.signatures_gas.sum += gas_data.signatures;
  144. // Update the statistics related to deployment gas
  145. self.deployments_gas.sum += gas_data.deployments;
  146. if self.txs_count == 1 {
  147. // For the first transaction, set min/max to the transaction values
  148. self.total_gas.min = gas_data.total_gas_used();
  149. self.total_gas.max = gas_data.total_gas_used();
  150. self.wasm_gas.min = gas_data.wasm;
  151. self.wasm_gas.max = gas_data.wasm;
  152. self.zk_circuits_gas.min = gas_data.zk_circuits;
  153. self.zk_circuits_gas.max = gas_data.zk_circuits;
  154. self.signatures_gas.min = gas_data.signatures;
  155. self.signatures_gas.max = gas_data.signatures;
  156. self.deployments_gas.min = gas_data.deployments;
  157. self.deployments_gas.max = gas_data.deployments;
  158. return;
  159. }
  160. // For subsequent transactions, compare with min/max
  161. self.total_gas.min = self.total_gas.min.min(gas_data.total_gas_used());
  162. self.total_gas.max = self.total_gas.max.max(gas_data.total_gas_used());
  163. self.wasm_gas.min = self.wasm_gas.min.min(gas_data.wasm);
  164. self.wasm_gas.max = self.wasm_gas.max.max(gas_data.wasm);
  165. self.zk_circuits_gas.min = self.zk_circuits_gas.min.min(gas_data.zk_circuits);
  166. self.zk_circuits_gas.max = self.zk_circuits_gas.max.max(gas_data.zk_circuits);
  167. self.signatures_gas.min = self.signatures_gas.min.min(gas_data.signatures);
  168. self.signatures_gas.max = self.signatures_gas.max.max(gas_data.signatures);
  169. self.deployments_gas.min = self.deployments_gas.min.min(gas_data.deployments);
  170. self.deployments_gas.max = self.deployments_gas.max.max(gas_data.deployments);
  171. }
  172. }
  173. }
  174. /// Debug formatting support for [`GasMetrics`] instances to include averages.
  175. impl fmt::Debug for GasMetrics {
  176. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  177. f.debug_struct("GasMetrics")
  178. .field("txs_count", &self.txs_count)
  179. .field("avg_total_gas_used", &self.avg_total_gas_used())
  180. .field("avg_wasm_gas_used", &self.avg_wasm_gas_used())
  181. .field("avg_zk_circuits_gas_used", &self.avg_zk_circuits_gas_used())
  182. .field("avg_signatures_gas_used", &self.avg_signatures_gas_used())
  183. .field("avg_deployments_gas_used", &self.avg_deployments_gas_used())
  184. .field("total_gas", &format_args!("{:?}", self.total_gas))
  185. .field("wasm_gas", &format_args!("{:?}", self.wasm_gas))
  186. .field("zk_circuits_gas", &format_args!("{:?}", self.zk_circuits_gas))
  187. .field("signatures_gas", &format_args!("{:?}", self.signatures_gas))
  188. .field("deployments_gas", &format_args!("{:?}", self.deployments_gas))
  189. .field("timestamp", &self.timestamp)
  190. .finish()
  191. }
  192. }
  193. /// The `MetricStore` serves as the entry point for managing metrics,
  194. /// offering an API for fetching, inserting, and resetting metrics backed by a Sled database.
  195. ///
  196. /// It organizes data into separate Sled trees, including main storage for gas metrics by a defined time interval,
  197. /// a tree containing metrics by height for handling reorgs, and a transaction-specific gas data tree.
  198. /// Different keys, such as gas metric keys, block heights, and transaction hashes, are used to handle
  199. /// various use cases.
  200. ///
  201. /// The `MetricStore` utilizes an overlay pattern for write operations, allowing unified management of metrics,
  202. /// by internally delegating write-related actions like adding metrics and handling reorgs to [`MetricsStoreOverlay`].
  203. #[derive(Clone)]
  204. pub struct MetricsStore {
  205. /// Pointer to the underlying sled database used by the store and its associated overlay
  206. pub sled_db: sled::Db,
  207. /// Primary sled tree for storing gas metrics, utilizing [`GasMetricsKey`] as keys and
  208. /// serialized [`GasMetrics`] as values.
  209. pub main: sled::Tree,
  210. /// Sled tree for storing gas metrics by height, utilizing block `height` as keys
  211. /// and serialized [`GasMetrics`] as values.
  212. pub by_height: sled::Tree,
  213. /// Sled tree for storing transaction gas data, utilizing [`TransactionHash`] inner value as keys
  214. /// and serialized [`GasData`] as values.
  215. pub tx_gas_data: sled::Tree,
  216. }
  217. // Temporarily disable unused warnings until the store is integrated with the explorer
  218. #[allow(dead_code)]
  219. impl MetricsStore {
  220. /// Creates a [`MetricsStore`] instance by opening the necessary trees in the provided sled database [`Db`]
  221. pub fn new(db: &sled::Db) -> Result<Self> {
  222. let main = db.open_tree(SLED_GAS_METRICS_TREE)?;
  223. let tx_gas_data = db.open_tree(SLED_TX_GAS_DATA_TREE)?;
  224. let metrcs_by_height = db.open_tree(SLED_GAS_METRICS_BY_HEIGHT_TREE)?;
  225. Ok(Self { sled_db: db.clone(), main, tx_gas_data, by_height: metrcs_by_height })
  226. }
  227. /// Fetches [`GasMetrics`]s associated with the provided slice of [`GasMetricsKey`]s.
  228. pub fn get(&self, keys: &[GasMetricsKey]) -> Result<Vec<GasMetrics>> {
  229. let mut ret = Vec::with_capacity(keys.len());
  230. for key in keys {
  231. if let Some(metrics_bytes) = self.main.get(key.to_sled_key())? {
  232. let metrics = deserialize(&metrics_bytes).map_err(Error::from)?;
  233. ret.push(metrics);
  234. }
  235. }
  236. Ok(ret)
  237. }
  238. /// Fetches [`GasMetrics`]s associated with the provided slice of [`u32`] heights.
  239. pub fn get_by_height(&self, heights: &[u32]) -> Result<Vec<GasMetrics>> {
  240. let mut ret = Vec::with_capacity(heights.len());
  241. for height in heights {
  242. if let Some(metrics_bytes) = self.by_height.get(height.to_be_bytes())? {
  243. let metrics = deserialize(&metrics_bytes).map_err(Error::from)?;
  244. ret.push(metrics);
  245. }
  246. }
  247. Ok(ret)
  248. }
  249. /// Fetches the most recent [`GasMetrics`] and its associated [`GasMetricsKey`] from the main tree,
  250. /// returning `None` if no metrics are found.
  251. pub fn get_last(&self) -> Result<Option<(GasMetricsKey, GasMetrics)>> {
  252. self.main
  253. .last()?
  254. .map(|(key_bytes, metrics_bytes)| {
  255. // Deserialize gas metrics key and value
  256. let key = GasMetricsKey::from_sled_key(&key_bytes)?;
  257. let metrics: GasMetrics = deserialize(&metrics_bytes).map_err(Error::from)?;
  258. debug!(target: "explorerd::metrics_store::get_last", "Deserialized metrics at key {key}: {metrics:?}");
  259. Ok((key, metrics))
  260. })
  261. .transpose()
  262. }
  263. /// Fetches all [`GasMetrics`] from the main tree without corresponding key, returning an empty `Vec`
  264. /// if no metrics are found.
  265. pub fn get_all_metrics(&self) -> Result<Vec<GasMetrics>> {
  266. // Iterate through all metrics, deserialize each one, and collect results
  267. self.main
  268. .iter()
  269. .map(|iter_result| match iter_result {
  270. Ok((_, metrics_bytes)) => deserialize(&metrics_bytes).map_err(Error::from),
  271. Err(e) => Err(Error::from(e)),
  272. })
  273. .collect()
  274. }
  275. /// Fetches the most recent [`GasMetrics`] and its associated `height` from the `by_height` tree, returning `None` if no metrics are found.
  276. pub fn get_last_by_height(&self) -> Result<Option<(u32, GasMetrics)>> {
  277. self.by_height
  278. .last()?
  279. .map(|(height_bytes, metrics_bytes)| {
  280. // Deserialize height key and value
  281. let key_bytes: [u8; 4] = height_bytes.as_ref().try_into().unwrap();
  282. let height = u32::from_be_bytes(key_bytes);
  283. let metrics: GasMetrics = deserialize(&metrics_bytes).map_err(Error::from)?;
  284. debug!(target: "explorerd::metrics_store::get_last_by_height", "Deserialized metrics at height {height:?}: {metrics:?}");
  285. Ok((height, metrics))
  286. })
  287. .transpose()
  288. }
  289. /// Fetches the [`GasData`] associated with the provided [`TransactionHash`], or `None` if no gas data is found.
  290. pub fn get_tx_gas_data(&self, tx_hash: &TransactionHash) -> Result<Option<GasData>> {
  291. // Query transaction gas data tree using provided hash
  292. let opt = self.tx_gas_data.get(tx_hash.inner())?;
  293. // Deserialize gas data, map error if needed, return result
  294. opt.map(|value| deserialize(&value).map_err(Error::from)).transpose()
  295. }
  296. /// Adds gas metrics for a specific block of transactions to the store.
  297. ///
  298. /// This function takes block `height`, [`Timestamp`], with associated pairs of [`TransactionHash`] and [`GasData`],
  299. /// and updates the accumulated gas metrics in the store. It handles the storage of metrics for both regular use and
  300. /// blockchain reorganizations.
  301. ///
  302. /// Delegates operation to [`MetricsStoreOverlay::insert_gas_metrics`], whose documentation
  303. /// provides more details.
  304. pub fn insert_gas_metrics(
  305. &self,
  306. block_height: u32,
  307. block_timestamp: &Timestamp,
  308. tx_hashes: &[TransactionHash],
  309. tx_gas_data: &[GasData],
  310. ) -> Result<GasMetricsKey> {
  311. let overlay = MetricsStoreOverlay::new(self.sled_db.clone())?;
  312. overlay.insert_gas_metrics(block_height, block_timestamp, tx_hashes, tx_gas_data)
  313. }
  314. /// Resets the gas metrics in the store to a specified `height` [`u32`].
  315. ///
  316. /// This function reverts all gas metrics data after the given height, effectively
  317. /// undoing changes made beyond that point. It's useful for handling blockchain
  318. /// reorganizations.
  319. ///
  320. /// Delegates operation to [`MetricsStoreOverlay::reset_gas_metrics`], whose documentation
  321. /// provides more details.
  322. pub fn reset_gas_metrics(&self, height: u32) -> Result<()> {
  323. let overlay = MetricsStoreOverlay::new(self.sled_db.clone())?;
  324. overlay.reset_gas_metrics(height)
  325. }
  326. /// Checks if provided [`GasMetricsKey`] exists in the store's main tree.
  327. pub fn contains(&self, key: &GasMetricsKey) -> Result<bool> {
  328. Ok(self.main.contains_key(key.to_sled_key())?)
  329. }
  330. /// Provides the number of stored metrics in the main tree.
  331. pub fn len(&self) -> usize {
  332. self.main.len()
  333. }
  334. /// Provides the number of stored metrics by height.
  335. pub fn len_by_height(&self) -> usize {
  336. self.by_height.len()
  337. }
  338. /// Returns the number of transaction gas usage metrics stored.
  339. pub fn len_tx_gas_data(&self) -> usize {
  340. self.tx_gas_data.len()
  341. }
  342. /// Checks if there are any gas metrics stored.
  343. pub fn is_empty(&self) -> bool {
  344. self.main.is_empty()
  345. }
  346. /// Checks if transaction gas data metrics are stored.
  347. pub fn is_empty_tx_gas_data(&self) -> bool {
  348. self.tx_gas_data.is_empty()
  349. }
  350. }
  351. /// The `MetricsStoreOverlay` provides write operations for managing metrics in conjunction with the
  352. /// underlying sled database. It supports inserting new [`GasData`] into the stored accumulated metrics,
  353. /// adding transaction gas data, and reverting metric changes after a specified height.
  354. struct MetricsStoreOverlay {
  355. /// Pointer to the overlay used for accessing and performing database write operations to the store.
  356. overlay: SledDbOverlayPtr,
  357. /// Pointer managed by the [`MetricsStore`] that references the sled instance on which the overlay operates.
  358. db: sled::Db,
  359. }
  360. impl MetricsStoreOverlay {
  361. /// Instantiate a [`MetricsStoreOverlay`] over the provided [`SledDbPtr`] instance.
  362. pub fn new(db: sled::Db) -> Result<Self> {
  363. // Create overlay pointer
  364. let overlay = Arc::new(Mutex::new(SledDbOverlay::new(&db, vec![])));
  365. // Open trees
  366. overlay.lock().unwrap().open_tree(SLED_GAS_METRICS_TREE, true)?;
  367. overlay.lock().unwrap().open_tree(SLED_GAS_METRICS_BY_HEIGHT_TREE, true)?;
  368. overlay.lock().unwrap().open_tree(SLED_TX_GAS_DATA_TREE, true)?;
  369. Ok(Self { overlay: overlay.clone(), db })
  370. }
  371. /// Adds the provided [`TransactionHash`] and [`GasData`] pairs to the accumulated [`GasMetrics`]
  372. /// in the store's [`SLED_GAS_METRICS_BY_HEIGHT_TREE`] and [`SLED_GAS_METRICS_TREE`] trees, while
  373. /// also storing transaction gas data in the [`SLED_TX_GAS_DATA_TREE`], committing all changes upon success.
  374. ///
  375. /// This function retrieves the latest recorded metrics, updates them with the new gas data, and
  376. /// stores the accumulated result. It uses the provided `block_timestamp` to create a normalied time-sequenced
  377. /// [`GasMetricsKey`] for metrics storage. The `block_height` is used as a key to store metrics by height
  378. /// which are used to handle chain reorganizations. After updating the aggregate metrics, it stores
  379. /// the transaction gas data for each transaction in the block.
  380. ///
  381. /// Returns the created [`GasMetricsKey`] that can be used to retrieve the metric upon success.
  382. pub fn insert_gas_metrics(
  383. &self,
  384. block_height: u32,
  385. block_timestamp: &Timestamp,
  386. tx_hashes: &[TransactionHash],
  387. tx_gas_data: &[GasData],
  388. ) -> Result<GasMetricsKey> {
  389. // Ensure lengths of tx_hashes and gas_data arrays match
  390. if tx_hashes.len() != tx_gas_data.len() {
  391. return Err(Error::Custom(String::from(
  392. "The lengths of tx_hashes and gas_data arrays must match",
  393. )));
  394. }
  395. // Ensure gas data is provided
  396. if tx_gas_data.is_empty() {
  397. return Err(Error::Custom(String::from("No transaction gas data was provided")));
  398. }
  399. // Lock the database
  400. let mut lock = self.overlay.lock().unwrap();
  401. // Retrieve latest recorded metrics, returning default if not exist
  402. let mut metrics = match self.get_last_by_height(&mut lock)? {
  403. None => GasMetrics::default(),
  404. Some((_, metrics)) => metrics,
  405. };
  406. // Update the accumulated metrics with the provided transaction gas data
  407. metrics.add(tx_gas_data);
  408. // Update the time that the metrics was recorded
  409. metrics.timestamp = *block_timestamp;
  410. // Insert metrics by height
  411. self.insert_by_height(&[block_height], &[metrics.clone()], &mut lock)?;
  412. // Create metrics key based on block_timestamp
  413. let metrics_key = GasMetricsKey::new(block_timestamp)?;
  414. // Normalize metric timestamp based on the key's time interval
  415. metrics.timestamp = GasMetricsKey::normalize_timestamp(block_timestamp)?;
  416. // Insert the gas metrics using metrics key
  417. self.insert(slice::from_ref(&metrics_key), &[metrics], &mut lock)?;
  418. // Insert the transaction gas data for each transaction in the block
  419. self.insert_tx_gas_data(tx_hashes, tx_gas_data, &mut lock)?;
  420. // Commit the changes
  421. lock.apply()?;
  422. Ok(metrics_key)
  423. }
  424. /// Inserts [`TransactionHash`] and [`GasData`] pairs into the store's [`SLED_TX_GAS_DATA_TREE`],
  425. /// committing the changes upon success.
  426. ///
  427. /// This function locks the overlay, verifies that the tx_hashes and gas_data arrays have matching lengths,
  428. /// then inserts them into the store while handling serialization and potential errors. Returns a
  429. /// successful result upon success.
  430. fn insert_tx_gas_data(
  431. &self,
  432. tx_hashes: &[TransactionHash],
  433. gas_data: &[GasData],
  434. lock: &mut MutexGuard<SledDbOverlay>,
  435. ) -> Result<()> {
  436. // Ensure lengths of tx_hashes and gas_data arrays match
  437. if tx_hashes.len() != gas_data.len() {
  438. return Err(Error::Custom(String::from(
  439. "The lengths of tx_hashes and gas_data arrays must match",
  440. )));
  441. }
  442. // Insert each transaction hash and gas data pair
  443. for (tx_hash, gas_data) in tx_hashes.iter().zip(gas_data.iter()) {
  444. // Serialize the gas data
  445. let serialized_gas_data = serialize(gas_data);
  446. // Insert serialized gas data
  447. lock.insert(SLED_TX_GAS_DATA_TREE, tx_hash.inner(), &serialized_gas_data)?;
  448. debug!(target: "explorerd::metrics_store::insert_tx_gas_data", "Inserted gas data for transaction {tx_hash}: {gas_data:?}");
  449. }
  450. Ok(())
  451. }
  452. /// Resets gas metrics in the [`SLED_GAS_METRICS_TREE`] and [`SLED_GAS_METRICS_BY_HEIGHT_TREE`]
  453. /// to a specified block height, undoing all entries after provided height and committing the
  454. /// changes upon success.
  455. ///
  456. /// This function first obtains a lock on the overlay, then reverts changes by calling
  457. /// [`Self::revert_by_height_metrics`] and [`Self::revert_metrics`]. Upon successful revert,
  458. /// all modifications made after the specified height are permanently reverted.
  459. pub fn reset_gas_metrics(&self, height: u32) -> Result<()> {
  460. // Obtain lock
  461. let mut lock = self.overlay.lock().unwrap();
  462. // Revert the metrics by height
  463. self.revert_by_height_metrics(height, &mut lock)?;
  464. // Revert the main metrics entries now that `by_height` tree is reset
  465. self.revert_metrics(&mut lock)?;
  466. // Commit the changes
  467. lock.apply()?;
  468. Ok(())
  469. }
  470. /// Inserts [`GasMetricsKey`] and [`GasMetrics`] pairs into the store's [`SLED_GAS_METRICS_TREE`].
  471. ///
  472. /// This function verifies that the provided keys and metrics arrays have matching lengths,
  473. /// then inserts each pair while handling serialization. Returns a successful result
  474. /// if all insertions are completed without errors.
  475. fn insert(
  476. &self,
  477. keys: &[GasMetricsKey],
  478. metrics: &[GasMetrics],
  479. lock: &mut MutexGuard<SledDbOverlay>,
  480. ) -> Result<()> {
  481. // Ensure lengths of keys and metrics match
  482. if keys.len() != metrics.len() {
  483. return Err(Error::Custom(String::from(
  484. "The lengths of keys and metrics arrays must match",
  485. )));
  486. }
  487. // Insert each metric corresponding to respective gas metrics key
  488. for (key, metric) in keys.iter().zip(metrics.iter()) {
  489. // Insert metric
  490. lock.insert(SLED_GAS_METRICS_TREE, &key.to_sled_key(), &serialize(metric))?;
  491. debug!(target: "explorerd::metrics_store::insert", "Added gas metrics using key {key}: {metric:?}");
  492. }
  493. Ok(())
  494. }
  495. /// Inserts provided [`u32`] height and [`GasMetrics`] pairs into the store's [`SLED_GAS_METRICS_BY_HEIGHT_TREE`].
  496. ///
  497. /// This function verifies matching lengths of provided heights and metrics arrays,
  498. /// and inserts each pair while handling serialization and errors. Returns a successful result
  499. /// if all insertions are completed without errors.
  500. fn insert_by_height(
  501. &self,
  502. heights: &[u32],
  503. metrics: &[GasMetrics],
  504. lock: &mut MutexGuard<SledDbOverlay>,
  505. ) -> Result<()> {
  506. // Ensure lengths of heights and metrics match
  507. if heights.len() != metrics.len() {
  508. return Err(Error::Custom(String::from(
  509. "The lengths of heights and metrics arrays must match",
  510. )));
  511. }
  512. // Insert each metric corresponding to respective height
  513. for (height, metric) in heights.iter().zip(metrics.iter()) {
  514. // Serialize the metric and handle potential errors
  515. let serialized_metric = serialize(metric);
  516. // Insert the serialized metric
  517. lock.insert(
  518. SLED_GAS_METRICS_BY_HEIGHT_TREE,
  519. &height.to_be_bytes(),
  520. &serialized_metric,
  521. )?;
  522. debug!(target: "explorerd::metrics_store::insert_by_height", "Added gas metrics using height {height}: {metric:?}");
  523. }
  524. Ok(())
  525. }
  526. /// This function reverts gas metric entries in the [`SLED_GAS_METRICS_TREE`] to align
  527. /// with the latest metrics state in the [`SLED_GAS_METRICS_BY_HEIGHT_TREE`].
  528. ///
  529. /// It first determines the target timestamp to revert to based on the latest entry
  530. /// in the by_height tree timestamp. Then, it iteratively removes entries from the main metrics
  531. /// tree that are newer than the target timestamp. Once all that is complete, it adds the latest
  532. /// metrics by height to the main metrics tree, returning a successful result if revert processes
  533. /// without error.
  534. fn revert_metrics(&self, lock: &mut MutexGuard<SledDbOverlay>) -> Result<()> {
  535. /*** Determine Metrics To Revert ***/
  536. // Get the last metrics by height and determine the target timestamp to revert to
  537. let latest_by_height = self.get_last_by_height(lock)?;
  538. let target_timestamp = match &latest_by_height {
  539. None => 0,
  540. Some((_, metrics)) => GasMetricsKey::normalize_timestamp(&metrics.timestamp)?.inner(),
  541. };
  542. // Get the timestamp of the latest metrics entry in the metrics store
  543. let mut current_timestamp = match self.get_last(lock)? {
  544. None => return Ok(()),
  545. Some((_, metrics)) => metrics.timestamp.inner(),
  546. };
  547. /*** Revert Main Tree Gas Metrics ***/
  548. // Iterate through at most the total number of gas metric tree entries
  549. for _ in 0..self.db.open_tree(SLED_GAS_METRICS_TREE)?.len() {
  550. // Stop the loop if the current timestamp is less than or equal to the target timestamp,
  551. // as there are no more entries to revert
  552. if current_timestamp <= target_timestamp {
  553. break;
  554. }
  555. // Create a `GasMetricsKey` for the current timestamp to locate the entry to be reverted.
  556. let key_to_revert = GasMetricsKey::new(current_timestamp)?;
  557. // Remove the corresponding entry from the gas metrics tree.
  558. lock.remove(SLED_GAS_METRICS_TREE, &key_to_revert.to_sled_key())?;
  559. info!(target: "explorerd:metrics_store:revert_metrics", "Successfully reverted metrics with key: {key_to_revert}");
  560. // Move to the previous valid timestamp by subtracting the defined time interval
  561. current_timestamp = current_timestamp.saturating_sub(GAS_METRICS_KEY_TIME_INTERVAL);
  562. }
  563. /*** Add the Latest Reverted Metrics To Main Tree ***/
  564. // Retrieve the latest metrics from the `by_height` tree and normalize its timestamp so it can be added to the main tree.
  565. // If there are no metrics in the `by_height` tree, we may have reset to 0, so return as there is nothing add.
  566. let latest_metrics = match latest_by_height {
  567. None => return Ok(()),
  568. Some((_, mut metrics)) => {
  569. metrics.timestamp = GasMetricsKey::normalize_timestamp(&metrics.timestamp)?;
  570. metrics
  571. }
  572. };
  573. // Add the latest metrics to the main tree based on latest reverted metrics by height
  574. let gas_metrics_key = GasMetricsKey::new(&latest_metrics.timestamp)?;
  575. self.insert(&[gas_metrics_key], &[latest_metrics], lock)?;
  576. Ok(())
  577. }
  578. /// Reverts gas metric entries from [`SLED_GAS_METRICS_BY_HEIGHT_TREE`] to provided `height`.
  579. ///
  580. /// This function iterates through the entries in gas metrics by height tree and removes all entries
  581. /// with heights greater than the specified `height`, effectively reverting all gas metrics beyond that point.
  582. fn revert_by_height_metrics(
  583. &self,
  584. height: u32,
  585. lock: &mut MutexGuard<SledDbOverlay>,
  586. ) -> Result<()> {
  587. // Retrieve the last stored block height
  588. let (last_height, _) = match self.get_last_by_height(lock)? {
  589. None => return Ok(()),
  590. Some(v) => v,
  591. };
  592. // Return early if the requested height is after the last stored height
  593. if height >= last_height {
  594. return Ok(());
  595. }
  596. // Remove keys greater than `height`
  597. while let Some((cur_height_bytes, _)) = lock.last(SLED_GAS_METRICS_BY_HEIGHT_TREE)? {
  598. // Convert height bytes to u32
  599. let cur_height = u32::from_be_bytes(cur_height_bytes.as_ref().try_into()?);
  600. // Process all heights that are bigger than provided `height`
  601. if cur_height <= height {
  602. break;
  603. }
  604. // Remove height being reverted
  605. lock.remove(SLED_GAS_METRICS_BY_HEIGHT_TREE, &cur_height_bytes)?;
  606. info!(target: "explorerd:metrics_store:revert_by_height_metrics", "Successfully reverted metrics with height: {cur_height}");
  607. }
  608. Ok(())
  609. }
  610. /// Fetches the most recent gas metrics from [`SLED_GAS_METRICS_TREE`], returning an option
  611. /// containing a metrics key [`GasMetricsKey`] and [`GasMetrics`] pair, or `None` if no metrics exist.
  612. fn get_last(
  613. &self,
  614. lock: &mut MutexGuard<SledDbOverlay>,
  615. ) -> Result<Option<(GasMetricsKey, GasMetrics)>> {
  616. // Fetch and deserialize key and metric pair
  617. lock.last(SLED_GAS_METRICS_TREE)?
  618. .map(|(key_bytes, metrics_bytes)| {
  619. // Deserialize the metrics key
  620. let key = GasMetricsKey::from_sled_key(&key_bytes)?;
  621. // Deserialize the stored gas metrics
  622. let metrics: GasMetrics = deserialize(&metrics_bytes).map_err(Error::from)?;
  623. Ok((key, metrics))
  624. })
  625. .transpose()
  626. }
  627. /// Fetches the most recent gas metrics from [`SLED_GAS_METRICS_BY_HEIGHT_TREE`], returning an option
  628. /// containing a height [`u32`] and [`GasMetrics`] pair, or `None` if no metrics exist.
  629. fn get_last_by_height(
  630. &self,
  631. lock: &mut MutexGuard<SledDbOverlay>,
  632. ) -> Result<Option<(u32, GasMetrics)>> {
  633. // Fetch and deserialize height and metric pair
  634. lock.last(SLED_GAS_METRICS_BY_HEIGHT_TREE)?
  635. .map(|(height_bytes, metrics_bytes)| {
  636. // Deserialize the height
  637. let key_bytes: [u8; 4] = height_bytes.as_ref().try_into().unwrap();
  638. let height = u32::from_be_bytes(key_bytes);
  639. // Deserialize the stored gas metrics
  640. let metrics: GasMetrics = deserialize(&metrics_bytes).map_err(Error::from)?;
  641. Ok((height, metrics))
  642. })
  643. .transpose()
  644. }
  645. }
  646. /// Represents a key used to store and fetch metrics in the metrics store.
  647. ///
  648. /// This struct provides methods for creating, serializing, and deserializing gas metrics keys.
  649. /// It supports creation from various time representations through the [`GasMetricsKeySource`] trait
  650. /// and offers conversion methods for use with a sled database.
  651. #[derive(Debug, Eq, PartialEq, Clone)]
  652. pub struct GasMetricsKey(pub DateTime);
  653. impl GasMetricsKey {
  654. /// Creates a new [`GasMetricsKey`] from a source that implements [`GasMetricsKeySource`].
  655. /// Depending on the use case, the key supports different input sources such as `Timestamp`, `u64` timestamp,
  656. /// or `&str` timestamp to create the key.
  657. pub fn new<T: GasMetricsKeySource>(source: T) -> Result<GasMetricsKey> {
  658. source.to_key()
  659. }
  660. /// Gets the inner [`DateTime`] value.
  661. pub fn inner(&self) -> &DateTime {
  662. &self.0
  663. }
  664. /// Converts the [`GasMetricsKey`] into a key suitable for use with a sled database.
  665. pub fn to_sled_key(&self) -> Vec<u8> {
  666. // Create a new vector with a capacity of 28 bytes
  667. let mut sled_key = Vec::with_capacity(28);
  668. // Push the byte representations of each field into the vector
  669. sled_key.extend_from_slice(&self.inner().year.to_be_bytes());
  670. sled_key.extend_from_slice(&self.inner().month.to_be_bytes());
  671. sled_key.extend_from_slice(&self.inner().day.to_be_bytes());
  672. sled_key.extend_from_slice(&self.inner().hour.to_be_bytes());
  673. sled_key.extend_from_slice(&self.inner().min.to_be_bytes());
  674. sled_key.extend_from_slice(&self.inner().sec.to_be_bytes());
  675. sled_key.extend_from_slice(&self.inner().nanos.to_be_bytes());
  676. // Return sled key
  677. sled_key
  678. }
  679. /// Converts a `sled` key into a [`GasMetricsKey`] by deserializing a slice of bytes.
  680. pub fn from_sled_key(bytes: &[u8]) -> Result<Self> {
  681. if bytes.len() != 28 {
  682. return Err(Error::Custom(String::from("Invalid byte length for GasMetricsKey")));
  683. }
  684. // Deserialize byte representations into each field
  685. let key = DateTime {
  686. year: u32::from_be_bytes(bytes[0..4].try_into()?),
  687. month: u32::from_be_bytes(bytes[4..8].try_into()?),
  688. day: u32::from_be_bytes(bytes[8..12].try_into()?),
  689. hour: u32::from_be_bytes(bytes[12..16].try_into()?),
  690. min: u32::from_be_bytes(bytes[16..20].try_into()?),
  691. sec: u32::from_be_bytes(bytes[20..24].try_into()?),
  692. nanos: u32::from_be_bytes(bytes[24..28].try_into()?),
  693. };
  694. Ok(Self(key))
  695. }
  696. /// Normalizes the given [`DateTime`] to the start of hour.
  697. pub fn normalize_date_time(date_time: DateTime) -> DateTime {
  698. DateTime {
  699. nanos: 0,
  700. sec: 0,
  701. min: 0,
  702. hour: date_time.hour,
  703. day: date_time.day,
  704. month: date_time.month,
  705. year: date_time.year,
  706. }
  707. }
  708. /// Normalizes a given [`Timestamp`] to the start of the hour.
  709. pub fn normalize_timestamp(timestamp: &Timestamp) -> Result<Timestamp> {
  710. let remainder = timestamp.inner() % GAS_METRICS_KEY_TIME_INTERVAL;
  711. timestamp.checked_sub(Timestamp::from_u64(remainder))
  712. }
  713. }
  714. impl fmt::Display for GasMetricsKey {
  715. fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
  716. write!(f, "{}", self.inner())
  717. }
  718. }
  719. /// Provides a unified method for creating new instances of GasMetricKeys using
  720. /// various time representations: [`Timestamp`], `u64` timestamp, or `&str` timestamp.
  721. pub trait GasMetricsKeySource {
  722. fn to_key(&self) -> Result<GasMetricsKey>;
  723. }
  724. /// Implements [`GasMetricsKeySource`] for &[`Timestamp`], converting it to a [`GasMetricsKey`].
  725. impl GasMetricsKeySource for &Timestamp {
  726. fn to_key(&self) -> Result<GasMetricsKey> {
  727. let date_time = DateTime::from_timestamp(self.inner(), 0);
  728. Ok(GasMetricsKey(GasMetricsKey::normalize_date_time(date_time)))
  729. }
  730. }
  731. /// Implements [`GasMetricsKeySource`] for `u64`, converting it to a [`GasMetricsKey`].
  732. impl GasMetricsKeySource for u64 {
  733. fn to_key(&self) -> Result<GasMetricsKey> {
  734. let date_time = DateTime::from_timestamp(*self, 0);
  735. Ok(GasMetricsKey(GasMetricsKey::normalize_date_time(date_time)))
  736. }
  737. }
  738. /// Implements [`GasMetricsKeySource`] for string slices, converting a `&str` in the `YYYY-MM-DD HH:mm:ss UTC` format
  739. /// to a [`GasMetricsKey`]. Returns an [`Error::ParseFailed`] error if the provided timestamp string slice is invalid.
  740. impl GasMetricsKeySource for &str {
  741. fn to_key(&self) -> Result<GasMetricsKey> {
  742. let date_time = DateTime::from_timestamp_str(self)?;
  743. Ok(GasMetricsKey(GasMetricsKey::normalize_date_time(date_time)))
  744. }
  745. }
  746. #[cfg(test)]
  747. /// This test module verifies the correct insertion, retrieval, and reset of metrics in the store.
  748. /// It covers adding metrics, searching metrics by time and transaction hash, and resetting metrics with specified heights.
  749. mod tests {
  750. use std::{
  751. str::FromStr,
  752. time::{Duration, SystemTime, UNIX_EPOCH},
  753. };
  754. use darkfi::util::{
  755. logger::{setup_test_logger, Level},
  756. time::DateTime,
  757. };
  758. use structopt::lazy_static::lazy_static;
  759. use tracing::warn;
  760. use super::*;
  761. /// Number of heights to simulate.
  762. const HEIGHT: u32 = 10;
  763. /// Fixed timestamp in seconds since UNIX epoch.
  764. const FIXED_TIMESTAMP: u64 = 1732042800;
  765. /// [`FIXED_TIMESTAMP`] timestamp as a string in UTC format.
  766. const FIXED_TIMESTAMP_STR: &str = "2024-11-19T19:00:00";
  767. lazy_static! {
  768. /// Test transaction hash.
  769. pub static ref TX_HASH: TransactionHash = TransactionHash::from_str(
  770. "92225ff00a3755d8df93c626b59f6e36cf021d85ebccecdedc38f3f1890a15fc"
  771. ).expect("Invalid transaction hash");
  772. }
  773. /// Tests inserting gas metrics, verifying the correctness of stored metrics.
  774. #[test]
  775. fn test_insert_gas_metrics() -> Result<()> {
  776. // Declare constants used for test
  777. const EXPECTED_HEIGHT: usize = HEIGHT as usize - 1;
  778. // Setup test, returning initialized metrics store
  779. let store = setup()?;
  780. // Load test data into the store and get the expected metrics results
  781. let test_data = load_random_metrics(&store, |_, _| {})?;
  782. // Verify metrics were inserted with the expected counts
  783. assert_eq!(store.len(), EXPECTED_HEIGHT);
  784. // Process height 0 test data separately
  785. let mut test_data_iter = test_data.iter();
  786. // For height 0, confirm there are no metrics stored in the store
  787. if let Some(test_data_height0) = test_data_iter.next() {
  788. let actual_height0 = store.get(&[GasMetricsKey::new(&test_data_height0.timestamp)?])?;
  789. assert!(
  790. actual_height0.is_empty(),
  791. "Timestamp associated with height 0 should not have any metrics stored"
  792. );
  793. }
  794. // Process remaining test data, verifying that each stored metric matches expected results
  795. for expected in test_data_iter {
  796. let actual = store.get(&[GasMetricsKey::new(&expected.timestamp)?])?;
  797. let expected_normalized = normalize_metrics_timestamp(expected)?;
  798. assert_eq!(&expected_normalized, &actual[0]);
  799. }
  800. Ok(())
  801. }
  802. /// Tests inserting gas metrics into the `by_height` tree, verifying the correctness of stored metrics.
  803. #[test]
  804. fn test_insert_by_height_gas_metrics() -> Result<()> {
  805. // Declare constants used for test
  806. const EXPECTED_HEIGHT: usize = HEIGHT as usize - 1;
  807. // Setup test, returning initialized metrics store
  808. let store = setup()?;
  809. // Load test data into the store and get the expected metrics results
  810. let test_data = load_random_metrics(&store, |_, _| {})?;
  811. // Verify metrics were inserted with the expected counts
  812. assert_eq!(store.len(), EXPECTED_HEIGHT);
  813. // For height 0, confirm there are no metrics stored in metrics by height
  814. let actual_height0 = store.get_by_height(&[0])?;
  815. assert!(actual_height0.is_empty(), "Height 0 should not have any metrics stored");
  816. // Process remaining heights, verifying that each stored metric matches expected results
  817. for (height, expected) in (1..).zip(test_data.iter().skip(1)) {
  818. let actual = store.get_by_height(&[height])?;
  819. assert!(!actual.is_empty(), "No metrics found for height {height}");
  820. assert_eq!(expected, &actual[0]);
  821. }
  822. Ok(())
  823. }
  824. /// Tests searching gas metrics by the hour, verifying the correct metrics are found
  825. /// and match expected values.
  826. #[test]
  827. fn test_search_metrics_by_hour() -> Result<()> {
  828. // Setup test, returning initialized metrics store
  829. let store = setup()?;
  830. // Load test data, initializing expected with the fourth loaded record
  831. let expected = &load_random_metrics(&store, |_, _| {})?[3];
  832. // Create search criteria based on the expected timestamp value
  833. let search_criteria = DateTime::from_timestamp(expected.timestamp.inner(), 0);
  834. // Search metrics by the hour
  835. let actual_opt = store.main.iter().find_map(|res| {
  836. res.ok().and_then(|(k, v)| {
  837. let key = GasMetricsKey::from_sled_key(&k).ok()?;
  838. if key.inner().hour == search_criteria.hour {
  839. deserialize::<GasMetrics>(&v).ok()
  840. } else {
  841. None
  842. }
  843. })
  844. });
  845. // Verify the found metrics match expected results
  846. assert!(actual_opt.is_some());
  847. assert_eq!(normalize_metrics_timestamp(expected)?, actual_opt.unwrap());
  848. Ok(())
  849. }
  850. /// Tests fetching gas metrics by a timestamp string, verifying the retrieved metrics
  851. /// match expected values.
  852. #[test]
  853. fn test_get_metrics_by_timestamp_str() -> Result<()> {
  854. // Setup test, returning initialized metrics store
  855. let store = setup()?;
  856. // Load fixed data needed for test, initializing expected with the first loaded record
  857. let (expected, _) = &load_fixed_metrics(&store)?[0];
  858. // Create gas metrics key using a test fixed timestamp
  859. let gas_metrics_key = GasMetricsKey::new(FIXED_TIMESTAMP_STR)?;
  860. // Verify the key retrieves the correct metrics and matches the expected value
  861. let actual = store.get(&[gas_metrics_key])?;
  862. assert_eq!(expected, &actual[0]);
  863. Ok(())
  864. }
  865. /// Tests the insertion and retrieval of transaction gas data in the store, verifying expected results.
  866. /// Additionally, it tests that transactions not found in the store correctly return a `None` result.
  867. #[test]
  868. fn test_tx_gas_data() -> Result<()> {
  869. let tx_hash_not_found: TransactionHash = TransactionHash::from_str(
  870. "93325ff00a3755d8df93c626b59f6e36cf021d85ebccecdedc38f3f1890a15fc",
  871. )
  872. .expect("Invalid hash");
  873. // Setup test, returning initialized metrics store
  874. let store = setup()?;
  875. // Load data needed for test, initializing expected with the first loaded record
  876. let (_, expected) = &load_fixed_metrics(&store)?[0];
  877. // Verify that existing transaction is found
  878. let actual_opt = store.get_tx_gas_data(&TX_HASH)?;
  879. assert!(actual_opt.is_some());
  880. assert_eq!(*expected, actual_opt.unwrap());
  881. // Verify that transactions that do not exist return None result
  882. let actual_not_found = store.get_tx_gas_data(&tx_hash_not_found)?;
  883. assert_eq!(None, actual_not_found);
  884. Ok(())
  885. }
  886. /// Tests resetting gas metrics within a specified height range, verifying that both the `by_height` and `main` trees
  887. /// are properly set to the reset height.
  888. #[test]
  889. fn test_reset_metrics_within_height_range() -> Result<()> {
  890. // Declare constants used for test
  891. const RESET_HEIGHT: u32 = 6;
  892. // Setup test, returning initialized metrics store
  893. let store = setup()?;
  894. // Load test data into the store and get the expected reset metrics result
  895. let expected = load_reset_metrics(&store, RESET_HEIGHT)?;
  896. // Reset metrics
  897. store.reset_gas_metrics(RESET_HEIGHT)?;
  898. // Fetch reset metrics by height
  899. let actual_by_height_opt = store.get_last_by_height()?;
  900. assert!(actual_by_height_opt.is_some(), "Expected get_last_by_height to return metrics");
  901. // Verify metrics by height are properly reset
  902. let (_, actual_by_height) = actual_by_height_opt.unwrap();
  903. assert_eq!(&expected, &actual_by_height);
  904. // Fetch reset main metrics
  905. let actual_main_opt = store.get_last()?;
  906. assert!(actual_main_opt.is_some(), "Expected get_last to return metrics");
  907. // Verify main metrics are properly reset
  908. let (_, actual_main_metrics) = actual_main_opt.unwrap();
  909. assert_eq!(&normalize_metrics_timestamp(&expected)?, &actual_main_metrics);
  910. Ok(())
  911. }
  912. /// Tests resetting the metrics store to height 0, ensuring it handles the operation gracefully without errors
  913. /// and verifies that no metrics remain in the store afterward.
  914. #[test]
  915. fn test_reset_metrics_height_to_0() -> Result<()> {
  916. // Declare constants used for test
  917. const RESET_HEIGHT: u32 = 0;
  918. const EXPECTED_RESET_HEIGHT: usize = 0;
  919. // Setup test, returning initialized metrics store
  920. let store = setup()?;
  921. // Load reset test data needed for test
  922. _ = load_reset_metrics(&store, RESET_HEIGHT)?;
  923. // Reset metrics
  924. store.reset_gas_metrics(RESET_HEIGHT)?;
  925. // Verify metrics were reset with the expected counts
  926. assert_eq!(store.len_by_height(), EXPECTED_RESET_HEIGHT);
  927. assert_eq!(store.len(), EXPECTED_RESET_HEIGHT);
  928. // Verify metrics by height are empty
  929. let actual_by_height_opt = store.get_last_by_height()?;
  930. assert!(actual_by_height_opt.is_none(), "Expected None from get_last_by_height");
  931. // Confirm main metrics are empty
  932. let actual_main_opt = store.get_last()?;
  933. assert!(actual_main_opt.is_none(), "Expected None from get_last");
  934. Ok(())
  935. }
  936. /// Tests that resetting beyond the number of available metrics does not change
  937. /// the store and no errors are thrown since there are no metrics to reset.
  938. #[test]
  939. fn test_reset_metrics_beyond_height() -> Result<()> {
  940. // Declare constants used for test
  941. const RESET_HEIGHT: u32 = HEIGHT + 1;
  942. const EXPECTED_RESET_HEIGHT: usize = HEIGHT as usize - 1;
  943. // Setup test, returning initialized metrics store
  944. let store = setup()?;
  945. // Load reset test data needed for test, storing the expected result
  946. let expected = load_reset_metrics(&store, RESET_HEIGHT)?;
  947. // Reset metrics to given height
  948. store.reset_gas_metrics(RESET_HEIGHT)?;
  949. // Verify metrics were reset with the expected counts
  950. assert_eq!(store.len_by_height(), EXPECTED_RESET_HEIGHT);
  951. assert_eq!(store.len(), EXPECTED_RESET_HEIGHT);
  952. // Verify that the last record for metrics by height is correctly reset
  953. let actual_by_height_opt = store.get_last_by_height()?;
  954. assert!(actual_by_height_opt.is_some(), "Expected get_last_by_height to return metrics");
  955. let (_, actual_by_height) = actual_by_height_opt.unwrap();
  956. assert_eq!(&expected, &actual_by_height);
  957. // Verify that the last record for main metrics is correctly reset
  958. let actual_main_opt = store.get_last()?;
  959. assert!(actual_main_opt.is_some(), "Expected get_last to return metrics");
  960. let (_, actual_main) = actual_main_opt.unwrap();
  961. assert_eq!(&normalize_metrics_timestamp(&expected)?, &actual_main);
  962. Ok(())
  963. }
  964. /// Tests resetting metrics at the last available height to verify that the code
  965. /// can handle the boundary condition.
  966. #[test]
  967. fn test_reset_metrics_at_height() -> Result<()> {
  968. // Declare constants used for test
  969. const RESET_HEIGHT: u32 = HEIGHT;
  970. const EXPECTED_RESET_HEIGHT: usize = HEIGHT as usize - 1;
  971. // Setup test, returning initialized metrics store
  972. let store = setup()?;
  973. // Load reset test data needed for test
  974. let expected = load_reset_metrics(&store, RESET_HEIGHT)?;
  975. // Reset metrics to given height
  976. store.reset_gas_metrics(RESET_HEIGHT)?;
  977. // Verify metrics were reset with the expected counts
  978. assert_eq!(store.len_by_height(), EXPECTED_RESET_HEIGHT);
  979. assert_eq!(store.len(), EXPECTED_RESET_HEIGHT);
  980. // Verify that the last record for metrics by height is correctly reset
  981. let actual_by_height_opt = store.get_last_by_height()?;
  982. assert!(actual_by_height_opt.is_some(), "Expected get_last_by_height to return metrics");
  983. let (_, actual_by_height) = actual_by_height_opt.unwrap();
  984. assert_eq!(&expected, &actual_by_height);
  985. // Verify that the last record for main metrics is correctly reset
  986. let actual_main_opt = store.get_last()?;
  987. assert!(actual_main_opt.is_some(), "Expected get_last to return metrics");
  988. let (_, actual_main) = actual_main_opt.unwrap();
  989. assert_eq!(&normalize_metrics_timestamp(&expected)?, &actual_main);
  990. Ok(())
  991. }
  992. /// Tests that resetting an empty metrics store gracefully handles
  993. /// the operation without errors and ensures the store remains empty.
  994. #[test]
  995. fn test_reset_empty_store() -> Result<()> {
  996. const RESET_HEIGHT: u32 = 6;
  997. // Setup test, returning initialized metrics store
  998. let store = setup()?;
  999. // Reset metrics with an empty store
  1000. store.reset_gas_metrics(RESET_HEIGHT)?;
  1001. // Verify no metrics with the expected counts
  1002. assert_eq!(store.len_by_height(), 0);
  1003. assert_eq!(store.len(), 0);
  1004. // Verify that metrics by height is empty
  1005. let actual_by_height = store.get_last_by_height()?;
  1006. assert!(actual_by_height.is_none(), "Expected get_last_by_height to return None");
  1007. // Verify main metrics is empty
  1008. let actual_main = store.get_last()?;
  1009. assert!(actual_main.is_none(), "Expected get_last to return None");
  1010. Ok(())
  1011. }
  1012. /// Sets up a test case for metrics store testing by initializing the logger,
  1013. /// creating a temporary database, and returning an initialized metrics store.
  1014. fn setup() -> Result<MetricsStore> {
  1015. // Initialize logger to show execution output
  1016. if setup_test_logger(
  1017. &["sled", "runtime", "net"],
  1018. false,
  1019. Level::Info,
  1020. //Level::Verbose,
  1021. //Level::Debug,
  1022. //Level::Trace,
  1023. )
  1024. .is_err()
  1025. {
  1026. warn!("Logger already initialized");
  1027. }
  1028. // Create a temporary directory for the sled database
  1029. let db =
  1030. sled::Config::new().temporary(true).open().expect("Unable to open test sled database");
  1031. // Initialize the metrics store
  1032. let metrics_store = MetricsStore::new(&db.clone())?;
  1033. Ok(metrics_store)
  1034. }
  1035. /// Loads random test gas metrics data into the given metrics store, simulating height 0 as a
  1036. /// genesis block with no metrics.
  1037. ///
  1038. /// Computes the starting block timestamp from the current system time for the first metric,
  1039. /// then inserts each subsequent metric at intervals of [`GAS_METRICS_KEY_TIME_INTERVAL`],
  1040. /// resulting in metrics being inserted one hour apart. The function iterates through a predefined
  1041. /// height range, as defined by [`HEIGHT`], to accumulate and insert gas metrics. After each
  1042. /// metric is stored, the `metric_loaded` closure is invoked, allowing the caller to perform
  1043. /// specific actions as the data is loaded.
  1044. ///
  1045. /// NOTE: A fixed transaction hash is used to insert the metrics, as this test data is solely intended
  1046. /// to validate gas metrics and not transaction-specific gas data.
  1047. ///
  1048. /// Upon success, it returns a list of snapshots of the accumulated metrics that were loaded.
  1049. fn load_random_metrics<F>(
  1050. metrics_store: &MetricsStore,
  1051. mut metrics_loaded: F,
  1052. ) -> Result<Vec<GasMetrics>>
  1053. where
  1054. F: FnMut(u32, &GasMetrics),
  1055. {
  1056. // Calculate the start block timestamp
  1057. let start_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
  1058. // Define variables to store accumulated loaded data
  1059. let mut accumulated_metrics = GasMetrics::default();
  1060. let mut metrics = Vec::with_capacity(HEIGHT as usize);
  1061. // Iterate and load data
  1062. for height in 0..HEIGHT {
  1063. let timestamp_secs = (UNIX_EPOCH +
  1064. Duration::from_secs(start_time + height as u64 * GAS_METRICS_KEY_TIME_INTERVAL))
  1065. .duration_since(UNIX_EPOCH)?
  1066. .as_secs();
  1067. // Initialize simulated block_timestamp
  1068. let block_timestamp = Timestamp::from(timestamp_secs);
  1069. accumulated_metrics.timestamp = block_timestamp;
  1070. // Simulate genesis block, metrics are stored after height 0
  1071. if height > 0 {
  1072. let tx_gas_data = random_gas_data(height as u64 + start_time);
  1073. accumulated_metrics.add(slice::from_ref(&tx_gas_data));
  1074. metrics_store.insert_gas_metrics(
  1075. height,
  1076. &block_timestamp,
  1077. &[*TX_HASH],
  1078. &[tx_gas_data],
  1079. )?;
  1080. }
  1081. // Invoke passed in metrics loaded closure
  1082. metrics_loaded(height, &accumulated_metrics);
  1083. // Add a snapshot of the accumulated metrics
  1084. metrics.push(accumulated_metrics.clone());
  1085. }
  1086. Ok(metrics)
  1087. }
  1088. /// Loads fixed test data into the metrics store using fixed timestamps,
  1089. /// returning snapshots of accumulated [`GasMetrics`] with corresponding [`GasData`]
  1090. /// used to update the metrics.
  1091. ///
  1092. /// Currently, this function only loads a single record but is designed to be extendable
  1093. /// to insert additional records in the future without affecting the method's return signature,
  1094. /// making it suitable for use in tests.
  1095. fn load_fixed_metrics(metrics_store: &MetricsStore) -> Result<Vec<(GasMetrics, GasData)>> {
  1096. // Convert the fixed timestamp constant to a Timestamp object
  1097. let fixed_timestamp = Timestamp::from_u64(FIXED_TIMESTAMP);
  1098. // Initialize an empty GasMetrics object to accumulate the data
  1099. let height: u32 = 1;
  1100. let mut accumulated_metrics = GasMetrics::default();
  1101. let mut metrics_vec = Vec::with_capacity(HEIGHT as usize);
  1102. // Initialize the block_timestamp using the fixed timestamp
  1103. let block_timestamp = fixed_timestamp;
  1104. accumulated_metrics.timestamp = block_timestamp;
  1105. // Generate random gas data for the given height
  1106. let gas_data = random_gas_data(height as u64);
  1107. accumulated_metrics.add(slice::from_ref(&gas_data));
  1108. // Insert the gas metrics into the metrics store
  1109. metrics_store.insert_gas_metrics(
  1110. height,
  1111. &block_timestamp,
  1112. &[*TX_HASH],
  1113. slice::from_ref(&gas_data),
  1114. )?;
  1115. metrics_vec.push((accumulated_metrics, gas_data));
  1116. Ok(metrics_vec)
  1117. }
  1118. /// Loads reset test data into the store, returning the accumulated gas metrics at the specified reset height.
  1119. fn load_reset_metrics(metrics_store: &MetricsStore, reset_height: u32) -> Result<GasMetrics> {
  1120. let mut reset_metrics = GasMetrics::default();
  1121. // Load metrics, passing in a closure to store the reset metrics
  1122. _ = load_random_metrics(metrics_store, |height, acc_metrics| {
  1123. // Store accumulated metrics at reset height
  1124. if reset_height == height || reset_height >= HEIGHT {
  1125. reset_metrics = acc_metrics.clone();
  1126. }
  1127. })?;
  1128. Ok(reset_metrics)
  1129. }
  1130. /// Generates random [`GasData`] based on the provided seed value, allowing for the simulation
  1131. /// of varied gas data values.
  1132. fn random_gas_data(seed: u64) -> GasData {
  1133. /// Defines a limit for gas data values.
  1134. const GAS_LIMIT: u64 = 100_000;
  1135. // Initialize gas usage with the provided seed
  1136. let mut gas_used = seed;
  1137. // Closure to generate a random gas value
  1138. let mut random_gas = || {
  1139. // Introduce variability using the seed and current gas_used
  1140. let variation = seed.wrapping_add(gas_used);
  1141. gas_used = gas_used.wrapping_mul(6364136223846793005).wrapping_add(variation);
  1142. gas_used
  1143. };
  1144. // Create GasData with random values constrained by GAS_LIMIT
  1145. GasData {
  1146. paid: random_gas() % GAS_LIMIT,
  1147. wasm: random_gas() % GAS_LIMIT,
  1148. zk_circuits: random_gas() % GAS_LIMIT,
  1149. signatures: random_gas() % GAS_LIMIT,
  1150. deployments: random_gas() % GAS_LIMIT,
  1151. }
  1152. }
  1153. /// Normalizes the [`GasMetrics`] timestamp to the start of the hour for test comparisons.
  1154. fn normalize_metrics_timestamp(metrics: &GasMetrics) -> Result<GasMetrics> {
  1155. let mut normalized_metrics = metrics.clone();
  1156. normalized_metrics.timestamp = GasMetricsKey::normalize_timestamp(&metrics.timestamp)?;
  1157. Ok(normalized_metrics)
  1158. }
  1159. }