openzeppelin_relayer/domain/transaction/evm/
status.rs

1//! This module contains the status-related functionality for EVM transactions.
2//! It includes methods for checking transaction status, determining when to resubmit
3//! or replace transactions with NOOPs, and updating transaction status in the repository.
4
5use alloy::network::ReceiptResponse;
6use alloy::primitives::{Address, TxKind};
7use alloy::rpc::types::{TransactionInput, TransactionRequest};
8use chrono::{DateTime, Duration, Utc};
9use eyre::Result;
10use serde_json::json;
11use tracing::{debug, error, info, warn};
12
13use super::super::common::is_active_nonce_status;
14use super::EvmRelayerTransaction;
15use super::{
16    ensure_status, evm_transaction::TX_NONCE_RECONCILE_TRIGGER, get_age_since_status_change,
17    has_enough_confirmations, is_noop, is_too_early_to_resubmit, is_transaction_valid, make_noop,
18    too_many_attempts, too_many_noop_attempts,
19};
20use crate::constants::{
21    get_evm_min_age_for_hash_recovery, get_evm_pending_recovery_trigger_timeout,
22    get_evm_prepare_timeout, get_evm_resend_timeout, ARBITRUM_TIME_TO_RESUBMIT,
23    DEFAULT_EVM_INCLUDE_REVERT_DATA, EVM_MIN_HASHES_FOR_RECOVERY, MAX_GAP_SCAN_RANGE,
24};
25use crate::domain::transaction::common::{
26    get_age_of_sent_at, is_final_state, is_pending_transaction,
27};
28use crate::domain::transaction::util::get_age_since_created;
29use crate::models::{EvmNetwork, NetworkRepoModel, NetworkType};
30use crate::repositories::{NetworkRepository, RelayerRepository};
31use crate::{
32    domain::transaction::evm::price_calculator::PriceCalculatorTrait,
33    jobs::{JobProducerTrait, StatusCheckContext},
34    models::{
35        EvmTransactionData, NetworkTransactionData, RelayerRepoModel, TransactionError,
36        TransactionRepoModel, TransactionStatus, TransactionUpdateRequest,
37    },
38    repositories::{Repository, TransactionCounterTrait, TransactionRepository},
39    services::{provider::EvmProviderTrait, signer::Signer},
40    utils::{get_resubmit_timeout_for_speed, get_resubmit_timeout_with_backoff},
41};
42
43/// The reason recorded on a failed EVM transaction when the on-chain revert payload cannot be
44/// recovered (or recovery is disabled). Kept byte-for-byte for consumers that match on it.
45const REVERT_REASON_GENERIC: &str = "Transaction reverted on-chain (receipt status: failed)";
46
47/// Upper bound (in characters) on the revert-data hex embedded in `status_reason`. Revert payloads
48/// are normally tiny, but they are RPC/contract-controlled, so we cap the recovered hex before it
49/// is persisted to avoid an oversized DB write or notification payload.
50const MAX_REVERT_DATA_HEX_LEN: usize = 4096;
51
52/// Caps an overlong revert-data hex string to [`MAX_REVERT_DATA_HEX_LEN`], appending a marker so
53/// consumers can tell the payload was clipped. The hex is ASCII, so slicing on a byte index is safe.
54fn truncate_revert_hex(hex: &str) -> String {
55    if hex.len() <= MAX_REVERT_DATA_HEX_LEN {
56        return hex.to_string();
57    }
58    format!("{}...(truncated)", &hex[..MAX_REVERT_DATA_HEX_LEN])
59}
60
61/// Reconstructs an `eth_call` request from persisted transaction data to reproduce the call as a
62/// state read. The fee fields (gas price / EIP-1559 caps) are carried through because contracts
63/// can branch on `tx.gasprice`, which would otherwise change the revert path versus the mined tx.
64fn build_revert_call_request(
65    evm_data: &EvmTransactionData,
66) -> Result<TransactionRequest, TransactionError> {
67    let from = evm_data.from.parse::<Address>().map_err(|e| {
68        TransactionError::UnexpectedError(format!("Invalid from address for revert recovery: {e}"))
69    })?;
70    let to = match evm_data.to.as_ref() {
71        Some(addr) => TxKind::Call(addr.parse::<Address>().map_err(|e| {
72            TransactionError::UnexpectedError(format!(
73                "Invalid to address for revert recovery: {e}"
74            ))
75        })?),
76        None => TxKind::Create,
77    };
78    let input = evm_data.data_to_bytes().map_err(|e| {
79        TransactionError::UnexpectedError(format!("Invalid input data for revert recovery: {e}"))
80    })?;
81
82    Ok(TransactionRequest {
83        from: Some(from),
84        to: Some(to),
85        value: Some(evm_data.value),
86        input: TransactionInput::from(input),
87        gas: evm_data.gas_limit,
88        gas_price: evm_data.gas_price,
89        max_fee_per_gas: evm_data.max_fee_per_gas,
90        max_priority_fee_per_gas: evm_data.max_priority_fee_per_gas,
91        ..Default::default()
92    })
93}
94
95/// Extracts the revert payload from a `callTracer` trace result. The top-level `output` field
96/// carries the revert bytes; an empty/`"0x"` value means no payload.
97fn extract_trace_output(trace: &serde_json::Value) -> Option<String> {
98    trace
99        .get("output")
100        .and_then(|v| v.as_str())
101        .filter(|s| s.starts_with("0x") && *s != "0x")
102        .map(|s| s.to_string())
103}
104
105impl<P, RR, NR, TR, J, S, TCR, PC> EvmRelayerTransaction<P, RR, NR, TR, J, S, TCR, PC>
106where
107    P: EvmProviderTrait + Send + Sync,
108    RR: RelayerRepository + Repository<RelayerRepoModel, String> + Send + Sync + 'static,
109    NR: NetworkRepository + Repository<NetworkRepoModel, String> + Send + Sync + 'static,
110    TR: TransactionRepository + Repository<TransactionRepoModel, String> + Send + Sync + 'static,
111    J: JobProducerTrait + Send + Sync + 'static,
112    S: Signer + Send + Sync + 'static,
113    TCR: TransactionCounterTrait + Send + Sync + 'static,
114    PC: PriceCalculatorTrait + Send + Sync,
115{
116    pub(super) async fn check_transaction_status(
117        &self,
118        tx: &TransactionRepoModel,
119    ) -> Result<TransactionStatus, TransactionError> {
120        // Early return if transaction is already in a final state
121        if is_final_state(&tx.status) {
122            return Ok(tx.status.clone());
123        }
124
125        // Early return for Pending/Sent states - these are DB-only states
126        // that don't require on-chain queries and may not have a hash yet
127        match tx.status {
128            TransactionStatus::Pending | TransactionStatus::Sent => {
129                return Ok(tx.status.clone());
130            }
131            _ => {}
132        }
133
134        let evm_data = tx.network_data.get_evm_transaction_data()?;
135        let tx_hash = evm_data
136            .hash
137            .as_ref()
138            .ok_or(TransactionError::UnexpectedError(
139                "Transaction hash is missing".to_string(),
140            ))?;
141
142        let receipt_result = self.provider().get_transaction_receipt(tx_hash).await?;
143
144        if let Some(receipt) = receipt_result {
145            if !receipt.inner.status() {
146                return Ok(TransactionStatus::Failed);
147            }
148            let last_block_number = self.provider().get_block_number().await?;
149            let tx_block_number = receipt
150                .block_number
151                .ok_or(TransactionError::UnexpectedError(
152                    "Transaction receipt missing block number".to_string(),
153                ))?;
154
155            let network_model = self
156                .network_repository()
157                .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
158                .await?
159                .ok_or(TransactionError::UnexpectedError(format!(
160                    "Network with chain id {} not found",
161                    evm_data.chain_id
162                )))?;
163
164            let network = EvmNetwork::try_from(network_model).map_err(|e| {
165                TransactionError::UnexpectedError(format!(
166                    "Error converting network model to EvmNetwork: {e}"
167                ))
168            })?;
169
170            if !has_enough_confirmations(
171                tx_block_number,
172                last_block_number,
173                network.required_confirmations,
174            ) {
175                debug!(
176                    tx_id = %tx.id,
177                    relayer_id = %tx.relayer_id,
178                    tx_hash = %tx_hash,
179                    "transaction mined but not confirmed"
180                );
181                return Ok(TransactionStatus::Mined);
182            }
183            Ok(TransactionStatus::Confirmed)
184        } else {
185            debug!(
186                tx_id = %tx.id,
187                relayer_id = %tx.relayer_id,
188                tx_hash = %tx_hash,
189                "transaction not yet mined"
190            );
191
192            // FALLBACK: Try to find transaction by checking all historical hashes
193            // Only do this for transactions that have multiple resubmission attempts
194            // and have been stuck in Submitted for a while
195            if tx.hashes.len() > 1 && self.should_try_hash_recovery(tx)? {
196                if let Some(recovered_tx) = self
197                    .try_recover_with_historical_hashes(tx, &evm_data)
198                    .await?
199                {
200                    // Return the status from the recovered (updated) transaction
201                    return Ok(recovered_tx.status);
202                }
203            }
204
205            Ok(TransactionStatus::Submitted)
206        }
207    }
208
209    /// Determines if a transaction should be resubmitted.
210    pub(super) async fn should_resubmit(
211        &self,
212        tx: &TransactionRepoModel,
213    ) -> Result<bool, TransactionError> {
214        // Validate transaction is in correct state for resubmission
215        ensure_status(tx, TransactionStatus::Submitted, Some("should_resubmit"))?;
216
217        let evm_data = tx.network_data.get_evm_transaction_data()?;
218        let age = get_age_of_sent_at(tx)?;
219
220        // Check if network lacks mempool and determine appropriate timeout
221        let network_model = self
222            .network_repository()
223            .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
224            .await?
225            .ok_or(TransactionError::UnexpectedError(format!(
226                "Network with chain id {} not found",
227                evm_data.chain_id
228            )))?;
229
230        let network = EvmNetwork::try_from(network_model).map_err(|e| {
231            TransactionError::UnexpectedError(format!(
232                "Error converting network model to EvmNetwork: {e}"
233            ))
234        })?;
235
236        let timeout = match network.is_arbitrum() {
237            true => ARBITRUM_TIME_TO_RESUBMIT,
238            false => get_resubmit_timeout_for_speed(&evm_data.speed),
239        };
240
241        let timeout_with_backoff = match network.is_arbitrum() {
242            true => timeout, // Use base timeout without backoff for Arbitrum
243            false => get_resubmit_timeout_with_backoff(timeout, tx.hashes.len()),
244        };
245
246        if age > Duration::milliseconds(timeout_with_backoff) {
247            debug!(
248                tx_id = %tx.id,
249                relayer_id = %tx.relayer_id,
250                age_ms = %age.num_milliseconds(),
251                "transaction has been pending for too long, resubmitting"
252            );
253            return Ok(true);
254        }
255        Ok(false)
256    }
257
258    /// Determines if a transaction should be replaced with a NOOP transaction.
259    ///
260    /// Returns a tuple `(should_noop, reason)` where:
261    /// - `should_noop`: `true` if transaction should be replaced with NOOP
262    /// - `reason`: Optional reason string explaining why NOOP is needed (only set when `should_noop` is `true`)
263    ///
264    /// # Arguments
265    ///
266    /// * `tx` - The transaction to check
267    pub(super) async fn should_noop(
268        &self,
269        tx: &TransactionRepoModel,
270    ) -> Result<(bool, Option<String>), TransactionError> {
271        if too_many_noop_attempts(tx) {
272            debug!("Transaction has too many NOOP attempts already");
273            return Ok((false, None));
274        }
275
276        let evm_data = tx.network_data.get_evm_transaction_data()?;
277        if is_noop(&evm_data) {
278            return Ok((false, None));
279        }
280
281        let network_model = self
282            .network_repository()
283            .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
284            .await?
285            .ok_or(TransactionError::UnexpectedError(format!(
286                "Network with chain id {} not found",
287                evm_data.chain_id
288            )))?;
289
290        let network = EvmNetwork::try_from(network_model).map_err(|e| {
291            TransactionError::UnexpectedError(format!(
292                "Error converting network model to EvmNetwork: {e}"
293            ))
294        })?;
295
296        if network.is_rollup() && too_many_attempts(tx) {
297            let reason =
298                "Rollup transaction has too many attempts. Replacing with NOOP.".to_string();
299            debug!(
300                tx_id = %tx.id,
301                relayer_id = %tx.relayer_id,
302                reason = %reason,
303                "replacing transaction with NOOP"
304            );
305            return Ok((true, Some(reason)));
306        }
307
308        if !is_transaction_valid(&tx.created_at, &tx.valid_until) {
309            let reason = "Transaction is expired. Replacing with NOOP.".to_string();
310            debug!(
311                tx_id = %tx.id,
312                relayer_id = %tx.relayer_id,
313                reason = %reason,
314                "replacing transaction with NOOP"
315            );
316            return Ok((true, Some(reason)));
317        }
318
319        if tx.status == TransactionStatus::Pending {
320            let created_at = &tx.created_at;
321            let created_time = DateTime::parse_from_rfc3339(created_at)
322                .map_err(|e| {
323                    TransactionError::UnexpectedError(format!("Invalid created_at timestamp: {e}"))
324                })?
325                .with_timezone(&Utc);
326            let age = Utc::now().signed_duration_since(created_time);
327            if age > get_evm_prepare_timeout() {
328                let reason = format!(
329                    "Transaction in Pending state for over {} minutes. Replacing with NOOP.",
330                    get_evm_prepare_timeout().num_minutes()
331                );
332                debug!(
333                    tx_id = %tx.id,
334                    relayer_id = %tx.relayer_id,
335                    reason = %reason,
336                    "replacing transaction with NOOP"
337                );
338                return Ok((true, Some(reason)));
339            }
340        }
341
342        let latest_block = self.provider().get_block_by_number().await;
343        if let Ok(block) = latest_block {
344            let block_gas_limit = block.header.gas_limit;
345            if let Some(gas_limit) = evm_data.gas_limit {
346                if gas_limit > block_gas_limit {
347                    let reason = format!(
348                                "Transaction gas limit ({gas_limit}) exceeds block gas limit ({block_gas_limit}). Replacing with NOOP.",
349                            );
350                    warn!(
351                        tx_id = %tx.id,
352                        tx_gas_limit = %gas_limit,
353                        block_gas_limit = %block_gas_limit,
354                        "transaction gas limit exceeds block gas limit, replacing with NOOP"
355                    );
356                    return Ok((true, Some(reason)));
357                }
358            }
359        }
360
361        Ok((false, None))
362    }
363
364    /// Helper method that updates transaction status only if it's different from the current status.
365    pub(super) async fn update_transaction_status_if_needed(
366        &self,
367        tx: TransactionRepoModel,
368        new_status: TransactionStatus,
369        status_reason: Option<String>,
370    ) -> Result<TransactionRepoModel, TransactionError> {
371        if tx.status != new_status {
372            return self
373                .update_transaction_status(tx, new_status, status_reason)
374                .await;
375        }
376        Ok(tx)
377    }
378
379    /// Prepares a NOOP transaction update request.
380    pub(super) async fn prepare_noop_update_request(
381        &self,
382        tx: &TransactionRepoModel,
383        is_cancellation: bool,
384        reason: Option<String>,
385    ) -> Result<TransactionUpdateRequest, TransactionError> {
386        let mut evm_data = tx.network_data.get_evm_transaction_data()?;
387        let network_model = self
388            .network_repository()
389            .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
390            .await?
391            .ok_or(TransactionError::UnexpectedError(format!(
392                "Network with chain id {} not found",
393                evm_data.chain_id
394            )))?;
395
396        let network = EvmNetwork::try_from(network_model).map_err(|e| {
397            TransactionError::UnexpectedError(format!(
398                "Error converting network model to EvmNetwork: {e}"
399            ))
400        })?;
401
402        make_noop(&mut evm_data, &network, Some(self.provider())).await?;
403
404        let noop_count = tx.noop_count.unwrap_or(0) + 1;
405        let update_request = TransactionUpdateRequest {
406            network_data: Some(NetworkTransactionData::Evm(evm_data)),
407            noop_count: Some(noop_count),
408            status_reason: reason,
409            is_canceled: if is_cancellation {
410                Some(true)
411            } else {
412                tx.is_canceled
413            },
414            ..Default::default()
415        };
416        Ok(update_request)
417    }
418
419    /// Handles transactions in the Submitted state.
420    ///
421    /// Before resubmitting, checks whether the transaction's nonce is ahead of
422    /// the on-chain nonce. If so, the tx can never mine because there's a gap
423    /// below it — schedules a nonce health job to fill the gap instead of
424    /// resubmitting (which would be futile).
425    async fn handle_submitted_state(
426        &self,
427        tx: TransactionRepoModel,
428    ) -> Result<TransactionRepoModel, TransactionError> {
429        if self.should_resubmit(&tx).await? {
430            // Before resubmitting, check if there's a nonce gap blocking this tx.
431            // Only worth the RPC call when we're already going to resubmit (tx is stale).
432            if let Some(nonce_gap_detected) = self.detect_nonce_gap_ahead(&tx).await {
433                if nonce_gap_detected {
434                    // Tx can't mine — nonce gap below it. Trigger health job, skip resubmit.
435                    return self
436                        .update_transaction_status_if_needed(tx, TransactionStatus::Submitted, None)
437                        .await;
438                }
439            }
440
441            let resubmitted_tx = self.handle_resubmission(tx).await?;
442            return Ok(resubmitted_tx);
443        }
444
445        self.update_transaction_status_if_needed(tx, TransactionStatus::Submitted, None)
446            .await
447    }
448
449    /// Checks whether the tx is blocked by a nonce gap below it.
450    ///
451    /// 1. Fetches on-chain nonce (single RPC call).
452    /// 2. If `tx_nonce <= on_chain_nonce` — no gap, tx should be mineable.
453    /// 3. Otherwise scans `on_chain_nonce..tx_nonce` using the Redis nonce index
454    ///    to check if every slot has an active (Pending/Sent/Submitted/Mined) tx.
455    /// 4. If any slot is empty or has a terminal-status tx — that's a real gap.
456    ///    Schedules a nonce health job and returns `Some(true)`.
457    ///
458    /// Returns:
459    /// - `Some(true)` — gap confirmed, health job scheduled
460    /// - `Some(false)` — no gap (all slots filled or tx is next)
461    /// - `None` — couldn't determine (missing nonce, RPC/Redis error)
462    async fn detect_nonce_gap_ahead(&self, tx: &TransactionRepoModel) -> Option<bool> {
463        let evm_data = match tx.network_data.get_evm_transaction_data() {
464            Ok(d) => d,
465            Err(_) => return None,
466        };
467        let tx_nonce = match evm_data.nonce {
468            Some(n) => n,
469            None => return None,
470        };
471
472        let on_chain_nonce = match self
473            .provider()
474            .get_transaction_count(&self.relayer().address)
475            .await
476        {
477            Ok(n) => n,
478            Err(e) => {
479                debug!(
480                    tx_id = %tx.id,
481                    error = %e,
482                    "nonce gap check: failed to get on-chain nonce, skipping"
483                );
484                return None;
485            }
486        };
487
488        // tx is the next expected nonce or already behind — no gap possible.
489        if tx_nonce <= on_chain_nonce {
490            return Some(false);
491        }
492
493        // Cap the scan to avoid an unbounded MGET if tx_nonce is very far ahead.
494        // If the gap is larger, we scan what we can — the health job's
495        // detect_nonce_gaps will use the nonce hint to extend its own scan.
496        let scan_to = std::cmp::min(tx_nonce, on_chain_nonce + MAX_GAP_SCAN_RANGE);
497
498        let occupancy = match self
499            .transaction_repository()
500            .get_nonce_occupancy(&tx.relayer_id, on_chain_nonce, scan_to)
501            .await
502        {
503            Ok(o) => o,
504            Err(e) => {
505                debug!(
506                    tx_id = %tx.id,
507                    error = %e,
508                    "nonce gap check: occupancy lookup failed, skipping"
509                );
510                return None;
511            }
512        };
513
514        let gap_nonces: Vec<u64> = occupancy
515            .into_iter()
516            .filter(|(_, status)| !status.as_ref().is_some_and(is_active_nonce_status))
517            .map(|(nonce, _)| nonce)
518            .collect();
519
520        if gap_nonces.is_empty() {
521            // All slots between on-chain and tx_nonce are actively filled — no gap.
522            return Some(false);
523        }
524
525        warn!(
526            tx_id = %tx.id,
527            relayer_id = %tx.relayer_id,
528            tx_nonce = tx_nonce,
529            on_chain_nonce = on_chain_nonce,
530            gap_count = gap_nonces.len(),
531            gaps = ?gap_nonces,
532            "nonce gaps confirmed below tx, scheduling nonce health to fill"
533        );
534
535        if let Err(e) = self.schedule_relayer_nonce_health_job(tx).await {
536            warn!(
537                tx_id = %tx.id,
538                error = %e,
539                "failed to schedule nonce health job for nonce gap"
540            );
541        }
542
543        Some(true)
544    }
545
546    /// Processes transaction resubmission logic
547    async fn handle_resubmission(
548        &self,
549        tx: TransactionRepoModel,
550    ) -> Result<TransactionRepoModel, TransactionError> {
551        debug!(
552            tx_id = %tx.id,
553            relayer_id = %tx.relayer_id,
554            status = ?tx.status,
555            "scheduling resubmit job for transaction"
556        );
557
558        // Check if transaction gas limit exceeds block gas limit before resubmitting
559        let (should_noop, reason) = self.should_noop(&tx).await?;
560        let tx_to_process = if should_noop {
561            self.process_noop_transaction(&tx, reason).await?
562        } else {
563            tx
564        };
565
566        self.send_transaction_resubmit_job(&tx_to_process).await?;
567        Ok(tx_to_process)
568    }
569
570    /// Handles NOOP transaction processing before resubmission
571    async fn process_noop_transaction(
572        &self,
573        tx: &TransactionRepoModel,
574        reason: Option<String>,
575    ) -> Result<TransactionRepoModel, TransactionError> {
576        debug!(
577            tx_id = %tx.id,
578            relayer_id = %tx.relayer_id,
579            status = ?tx.status,
580            "preparing transaction NOOP before resubmission"
581        );
582        let update = self.prepare_noop_update_request(tx, false, reason).await?;
583        let updated_tx = self
584            .transaction_repository()
585            .partial_update(tx.id.clone(), update)
586            .await?;
587
588        let res = self.send_transaction_update_notification(&updated_tx).await;
589        if let Err(e) = res {
590            error!(
591                tx_id = %updated_tx.id,
592                relayer_id = %updated_tx.relayer_id,
593                status = ?updated_tx.status,
594                error = %e,
595                "sending transaction update notification failed for NOOP transaction"
596            );
597        }
598        Ok(updated_tx)
599    }
600
601    /// Handles transactions in the Pending state.
602    async fn handle_pending_state(
603        &self,
604        tx: TransactionRepoModel,
605    ) -> Result<TransactionRepoModel, TransactionError> {
606        let (should_noop, reason) = self.should_noop(&tx).await?;
607        if should_noop {
608            // For Pending state transactions, nonces are not yet assigned, so we mark as Failed
609            // instead of NOOP. This matches prepare_transaction behavior.
610            debug!(
611                tx_id = %tx.id,
612                relayer_id = %tx.relayer_id,
613                reason = %reason.as_ref().unwrap_or(&"unknown".to_string()),
614                "marking pending transaction as Failed (nonce not assigned, no NOOP needed)"
615            );
616            let update = TransactionUpdateRequest {
617                status: Some(TransactionStatus::Failed),
618                status_reason: reason,
619                ..Default::default()
620            };
621            let updated_tx = self
622                .transaction_repository()
623                .partial_update(tx.id.clone(), update)
624                .await?;
625
626            let res = self.send_transaction_update_notification(&updated_tx).await;
627            if let Err(e) = res {
628                error!(
629                    tx_id = %updated_tx.id,
630                    relayer_id = %updated_tx.relayer_id,
631                    status = ?updated_tx.status,
632                    error = %e,
633                    "sending transaction update notification failed for Pending state NOOP"
634                );
635            }
636            return Ok(updated_tx);
637        }
638
639        // Check if transaction is stuck in Pending (prepare job may have failed)
640        let age = get_age_since_created(&tx)?;
641        if age > get_evm_pending_recovery_trigger_timeout() {
642            warn!(
643                tx_id = %tx.id,
644                relayer_id = %tx.relayer_id,
645                age_seconds = age.num_seconds(),
646                "transaction stuck in Pending, queuing prepare job"
647            );
648
649            // Re-queue prepare job
650            self.send_transaction_request_job(&tx).await?;
651        }
652
653        Ok(tx)
654    }
655
656    /// Handles transactions in the Mined state.
657    async fn handle_mined_state(
658        &self,
659        tx: TransactionRepoModel,
660    ) -> Result<TransactionRepoModel, TransactionError> {
661        self.update_transaction_status_if_needed(tx, TransactionStatus::Mined, None)
662            .await
663    }
664
665    /// Handles transactions in final states (Confirmed, Failed, Expired).
666    async fn handle_final_state(
667        &self,
668        tx: TransactionRepoModel,
669        status: TransactionStatus,
670        status_reason: Option<String>,
671    ) -> Result<TransactionRepoModel, TransactionError> {
672        self.update_transaction_status_if_needed(tx, status, status_reason)
673            .await
674    }
675
676    /// Marks a transaction as Failed with a given reason.
677    async fn mark_as_failed(
678        &self,
679        tx: TransactionRepoModel,
680        reason: String,
681    ) -> Result<TransactionRepoModel, TransactionError> {
682        warn!(
683            tx_id = %tx.id,
684            relayer_id = %tx.relayer_id,
685            reason = %reason,
686            "force-failing transaction due to circuit breaker"
687        );
688
689        let update = TransactionUpdateRequest {
690            status: Some(TransactionStatus::Failed),
691            status_reason: Some(reason),
692            ..Default::default()
693        };
694
695        let updated_tx = self
696            .transaction_repository()
697            .partial_update(tx.id.clone(), update)
698            .await?;
699
700        // Send notification (best effort)
701        if let Err(e) = self.send_transaction_update_notification(&updated_tx).await {
702            error!(
703                tx_id = %updated_tx.id,
704                relayer_id = %updated_tx.relayer_id,
705                error = %e,
706                "failed to send notification for force-failed transaction"
707            );
708        }
709
710        Ok(updated_tx)
711    }
712
713    /// Reconciles a single transaction's nonce state against on-chain reality.
714    ///
715    /// This is the fast-path reconciliation triggered by nonce errors during submission.
716    /// It checks:
717    /// 1. Receipt for current tx hash — if found, defers to normal flow
718    /// 2. Historical hash recovery — if a different hash was mined, updates the tx
719    /// 3. On-chain nonce comparison — if the nonce was consumed externally, marks Failed
720    ///
721    /// Returns `Some(tx)` if recovery handled the transaction (caller should return early),
722    /// or `None` to continue with normal status flow.
723    async fn reconcile_tx_nonce_state(
724        &self,
725        tx: &TransactionRepoModel,
726    ) -> Result<Option<TransactionRepoModel>, TransactionError> {
727        let evm_data = tx.network_data.get_evm_transaction_data()?;
728
729        // Track whether any RPC call failed transiently. If so, we must NOT
730        // make the irreversible "consumed externally" determination in step 3,
731        // because the tx could be mined under a hash we failed to check.
732        let mut had_rpc_errors = false;
733
734        // 1. Check receipt for current tx hash — if found, normal flow handles it
735        if let Some(ref hash) = evm_data.hash {
736            match self.provider().get_transaction_receipt(hash).await {
737                Ok(Some(_)) => {
738                    debug!(
739                        tx_id = %tx.id,
740                        hash = %hash,
741                        "nonce recovery: receipt found for current hash, deferring to normal flow"
742                    );
743                    return Ok(None);
744                }
745                Ok(None) => {
746                    // No receipt for current hash — continue recovery
747                }
748                Err(e) => {
749                    warn!(
750                        tx_id = %tx.id,
751                        hash = %hash,
752                        error = %e,
753                        "nonce recovery: error checking receipt for current hash"
754                    );
755                    had_rpc_errors = true;
756                }
757            }
758        }
759
760        // 2. Try historical hash recovery (reuse existing method)
761        if tx.hashes.len() > 1 {
762            match self.try_recover_with_historical_hashes(tx, &evm_data).await {
763                Ok(Some(recovered_tx)) => {
764                    debug!(
765                        tx_id = %tx.id,
766                        "nonce recovery: recovered transaction via historical hash"
767                    );
768                    return Ok(Some(recovered_tx));
769                }
770                Ok(None) => {
771                    // No historical hash found — continue
772                }
773                Err(e) => {
774                    warn!(
775                        tx_id = %tx.id,
776                        error = %e,
777                        "nonce recovery: error during historical hash recovery"
778                    );
779                    had_rpc_errors = true;
780                }
781            }
782        }
783
784        // 3. Compare on-chain nonce to determine if nonce was consumed externally.
785        //    Only safe to make this determination if all hash checks succeeded.
786        //    If any RPC call failed, the tx might be mined under a hash we couldn't check.
787        if had_rpc_errors {
788            warn!(
789                tx_id = %tx.id,
790                "nonce recovery: skipping nonce comparison due to RPC errors during hash checks, deferring to normal flow"
791            );
792            return Ok(None);
793        }
794
795        let tx_nonce = match evm_data.nonce {
796            Some(n) => n,
797            None => {
798                // No nonce assigned — can't compare, defer to normal flow
799                return Ok(None);
800            }
801        };
802
803        let on_chain_nonce = self
804            .provider()
805            .get_transaction_count(&self.relayer().address)
806            .await
807            .map_err(|e| {
808                TransactionError::UnexpectedError(format!(
809                    "Failed to get on-chain nonce for recovery: {e}"
810                ))
811            })?;
812
813        if on_chain_nonce > tx_nonce {
814            // Nonce was consumed but no known hash found — consumed externally
815            let reason = format!(
816                "Nonce {tx_nonce} consumed externally (on-chain nonce: {on_chain_nonce}). \
817                 No matching transaction hash found on-chain."
818            );
819            warn!(
820                tx_id = %tx.id,
821                relayer_id = %tx.relayer_id,
822                tx_nonce = tx_nonce,
823                on_chain_nonce = on_chain_nonce,
824                "nonce recovery: nonce consumed externally, marking as Failed"
825            );
826
827            let updated_tx = self
828                .update_transaction_status(tx.clone(), TransactionStatus::Failed, Some(reason))
829                .await?;
830
831            // External nonce consumption may have left the internal transaction counter
832            // behind the on-chain nonce, or created gaps. Schedule a nonce health job
833            // to sync the counter and fill any gaps with NOOPs. Best-effort — failure
834            // here doesn't block the recovery; the periodic health check will catch it.
835            if let Err(e) = self.schedule_relayer_nonce_health_job(tx).await {
836                warn!(
837                    tx_id = %tx.id,
838                    error = %e,
839                    "nonce recovery: failed to schedule nonce health after external consumption"
840                );
841            }
842
843            return Ok(Some(updated_tx));
844        }
845
846        // on_chain_nonce <= tx_nonce: nonce not yet consumed, defer to normal status flow
847        debug!(
848            tx_id = %tx.id,
849            tx_nonce = tx_nonce,
850            on_chain_nonce = on_chain_nonce,
851            "nonce recovery: on-chain nonce not past tx nonce, deferring to normal flow"
852        );
853        Ok(None)
854    }
855
856    /// Handles circuit breaker safely based on transaction status.
857    ///
858    /// This method implements the safe circuit breaker logic:
859    /// - **Pending/Sent**: Safe to mark as Failed (never broadcast to network)
860    /// - **Submitted**: Must trigger NOOP to clear nonce slot (regardless of expiry)
861    ///
862    /// For Submitted transactions, we always issue a NOOP because the nonce slot is
863    /// occupied and the original transaction could still execute. Simply marking as
864    /// Failed/Expired would leave the nonce blocked and risk the relayer stopping.
865    ///
866    /// Note: NOOP transactions are filtered out before entering this function.
867    async fn handle_circuit_breaker_safely(
868        &self,
869        tx: TransactionRepoModel,
870        ctx: &StatusCheckContext,
871    ) -> Result<TransactionRepoModel, TransactionError> {
872        let reason = format!(
873            "Transaction status monitoring failed after {} consecutive errors (total: {}). \
874             Last status: {:?}.",
875            ctx.consecutive_failures, ctx.total_failures, tx.status
876        );
877
878        match tx.status {
879            TransactionStatus::Pending => {
880                // Pending: no nonce assigned yet - safe to mark as Failed
881                debug!(
882                    tx_id = %tx.id,
883                    relayer_id = %tx.relayer_id,
884                    "circuit breaker: Pending transaction (no nonce) - safe to mark as Failed"
885                );
886                self.mark_as_failed(tx, reason).await
887            }
888            TransactionStatus::Sent => {
889                // Sent: nonce assigned but never broadcast to network.
890                // If a nonce is assigned, we must issue a NOOP to clear the nonce slot
891                // rather than just marking as Failed (which would leak the nonce).
892                let has_nonce = tx
893                    .network_data
894                    .get_evm_transaction_data()
895                    .map(|d| d.nonce.is_some())
896                    .unwrap_or(false);
897
898                if has_nonce {
899                    warn!(
900                        tx_id = %tx.id,
901                        relayer_id = %tx.relayer_id,
902                        "circuit breaker: Sent transaction with nonce assigned - triggering NOOP to clear nonce slot"
903                    );
904                    let noop_reason = Some(format!(
905                        "{reason}. Replacing with NOOP to clear nonce slot (Sent state with assigned nonce)."
906                    ));
907                    let updated_tx = self.process_noop_transaction(&tx, noop_reason).await?;
908                    // Must use resubmit (not submit) — resubmit re-signs with new gas pricing,
909                    // producing fresh `raw` bytes for the NOOP. submit_transaction would
910                    // broadcast the stale `raw` bytes which still contain the original tx.
911                    self.send_transaction_resubmit_job(&updated_tx).await?;
912                    Ok(updated_tx)
913                } else {
914                    // Defensive: Sent without nonce shouldn't normally happen
915                    debug!(
916                        tx_id = %tx.id,
917                        relayer_id = %tx.relayer_id,
918                        "circuit breaker: Sent transaction without nonce - safe to mark as Failed"
919                    );
920                    self.mark_as_failed(tx, reason).await
921                }
922            }
923            TransactionStatus::Submitted => {
924                // Submitted transactions occupy a nonce slot and could still execute.
925                // Regardless of expiry status, we MUST issue a NOOP to:
926                // 1. Clear the nonce slot so subsequent transactions can proceed
927                // 2. Prevent the original transaction from executing later
928                // Note: NOOP transactions are filtered out before entering this function.
929                warn!(
930                    tx_id = %tx.id,
931                    relayer_id = %tx.relayer_id,
932                    "circuit breaker: Submitted transaction - triggering NOOP to safely clear nonce"
933                );
934                let noop_reason = Some(format!(
935                    "{reason}. Replacing with NOOP to clear nonce slot."
936                ));
937                let updated_tx = self.process_noop_transaction(&tx, noop_reason).await?;
938                self.send_transaction_resubmit_job(&updated_tx).await?;
939                Ok(updated_tx)
940            }
941            _ => {
942                // Final states shouldn't reach here, but handle gracefully
943                debug!(
944                    tx_id = %tx.id,
945                    relayer_id = %tx.relayer_id,
946                    status = ?tx.status,
947                    "circuit breaker: unexpected status, returning transaction unchanged"
948                );
949                Ok(tx)
950            }
951        }
952    }
953
954    /// Inherent status-handling method.
955    ///
956    /// This method encapsulates the full logic for handling transaction status,
957    /// including resubmission, NOOP replacement, timeout detection, and updating status.
958    pub async fn handle_status_impl(
959        &self,
960        tx: TransactionRepoModel,
961        context: Option<StatusCheckContext>,
962    ) -> Result<TransactionRepoModel, TransactionError> {
963        debug!(
964            tx_id = %tx.id,
965            relayer_id = %tx.relayer_id,
966            status = ?tx.status,
967            "checking transaction status"
968        );
969
970        // 1. Early return if final state
971        if is_final_state(&tx.status) {
972            debug!(
973                tx_id = %tx.id,
974                relayer_id = %tx.relayer_id,
975                status = ?tx.status,
976                "transaction already in final state"
977            );
978            return Ok(tx);
979        }
980
981        // 1.1. Check if circuit breaker should force finalization
982        // Skip circuit breaker for NOOP transactions - they're already safe (just clearing nonce)
983        // and should be handled by normal status logic which will eventually resolve them.
984        if let Some(ref ctx) = context {
985            let is_noop_tx = tx
986                .network_data
987                .get_evm_transaction_data()
988                .map(|data| is_noop(&data))
989                .unwrap_or(false);
990
991            if ctx.should_force_finalize() && !is_noop_tx {
992                warn!(
993                    tx_id = %tx.id,
994                    consecutive_failures = ctx.consecutive_failures,
995                    total_failures = ctx.total_failures,
996                    max_consecutive = ctx.max_consecutive_failures,
997                    status = ?tx.status,
998                    "circuit breaker triggered - handling safely based on transaction state"
999                );
1000                return self.handle_circuit_breaker_safely(tx, ctx).await;
1001            }
1002
1003            if ctx.should_force_finalize() && is_noop_tx {
1004                debug!(
1005                    tx_id = %tx.id,
1006                    consecutive_failures = ctx.consecutive_failures,
1007                    relayer_id = %tx.relayer_id,
1008                    "circuit breaker would trigger but transaction is NOOP - continuing with normal status logic"
1009                );
1010            }
1011        }
1012
1013        // 1.2. Check for nonce recovery hint in job metadata (one-shot signal from submission errors).
1014        // This performs nonce reconciliation before normal status flow.
1015        // The hint is in job_metadata, not the transaction — subsequent retries won't have it.
1016        if let Some(ref ctx) = context {
1017            if let Some(ref metadata) = ctx.job_metadata {
1018                if let Some(hint) = metadata.get(TX_NONCE_RECONCILE_TRIGGER) {
1019                    debug!(
1020                        tx_id = %tx.id,
1021                        hint = %hint,
1022                        "nonce recovery hint detected - performing nonce reconciliation"
1023                    );
1024                    match self.reconcile_tx_nonce_state(&tx).await {
1025                        Ok(Some(recovered_tx)) => {
1026                            return Ok(recovered_tx);
1027                        }
1028                        Ok(None) => {
1029                            // Recovery didn't resolve it — fall through to normal flow
1030                            debug!(
1031                                tx_id = %tx.id,
1032                                "nonce recovery did not resolve transaction, continuing normal flow"
1033                            );
1034                        }
1035                        Err(e) => {
1036                            // Recovery failed — log and continue with normal flow
1037                            warn!(
1038                                tx_id = %tx.id,
1039                                error = %e,
1040                                "nonce recovery failed, falling through to normal status flow"
1041                            );
1042                        }
1043                    }
1044                }
1045            }
1046        }
1047
1048        // 2. Check transaction status first
1049        // This allows fast transactions to update their status immediately,
1050        // even if they're young (<20s). For Pending/Sent states, this returns
1051        // early without querying the blockchain.
1052        let status = self.check_transaction_status(&tx).await?;
1053
1054        debug!(
1055            tx_id = %tx.id,
1056            previous_status = ?tx.status,
1057            new_status = ?status,
1058            relayer_id = %tx.relayer_id,
1059            "transaction status check completed"
1060        );
1061
1062        // 2.1. Reload transaction from DB if status changed
1063        // This ensures we have fresh data if check_transaction_status triggered a recovery
1064        // or any other update that modified the transaction in the database.
1065        let tx = if status != tx.status {
1066            debug!(
1067                tx_id = %tx.id,
1068                old_status = ?tx.status,
1069                new_status = ?status,
1070                relayer_id = %tx.relayer_id,
1071                "status changed during check, reloading transaction from DB to ensure fresh data"
1072            );
1073            self.transaction_repository()
1074                .get_by_id(tx.id.clone())
1075                .await?
1076        } else {
1077            tx
1078        };
1079
1080        // 3. Check if too early for resubmission on in-progress transactions
1081        // For Pending/Sent/Submitted states, defer resubmission logic and timeout checks
1082        // if the transaction is too young. Just update status and return.
1083        // For other states (Mined/Confirmed/Failed/etc), process immediately regardless of age.
1084        if is_too_early_to_resubmit(&tx)? && is_pending_transaction(&status) {
1085            // Update status if it changed, then return
1086            return self
1087                .update_transaction_status_if_needed(tx, status, None)
1088                .await;
1089        }
1090
1091        // 4. Handle based on status (including complex operations like resubmission)
1092        match status {
1093            TransactionStatus::Pending => self.handle_pending_state(tx).await,
1094            TransactionStatus::Sent => self.handle_sent_state(tx).await,
1095            TransactionStatus::Submitted => self.handle_submitted_state(tx).await,
1096            TransactionStatus::Mined => self.handle_mined_state(tx).await,
1097            TransactionStatus::Failed => {
1098                // On the transition into Failed, attempt best-effort recovery of the on-chain
1099                // revert payload; on re-polls of an already-Failed tx, leave the reason untouched.
1100                let status_reason = if tx.status != TransactionStatus::Failed {
1101                    Some(self.build_failed_status_reason(&tx).await)
1102                } else {
1103                    None
1104                };
1105                self.handle_final_state(tx, status, status_reason).await
1106            }
1107            TransactionStatus::Confirmed
1108            | TransactionStatus::Expired
1109            | TransactionStatus::Canceled => self.handle_final_state(tx, status, None).await,
1110        }
1111    }
1112
1113    /// Builds the `status_reason` recorded when a transaction transitions into Failed from a
1114    /// failed on-chain receipt. When revert-data recovery is enabled (default) and a payload is
1115    /// recovered, returns the enriched reason; otherwise returns the generic string.
1116    async fn build_failed_status_reason(&self, tx: &TransactionRepoModel) -> String {
1117        if !self
1118            .relayer()
1119            .policies
1120            .get_evm_policy()
1121            .include_revert_data
1122            .unwrap_or(DEFAULT_EVM_INCLUDE_REVERT_DATA)
1123        {
1124            debug!(
1125                tx_id = %tx.id,
1126                "revert-data recovery disabled by policy; using generic reason"
1127            );
1128            return REVERT_REASON_GENERIC.to_string();
1129        }
1130
1131        match self.recover_revert_data(tx).await {
1132            Some(hex) => {
1133                let hex = truncate_revert_hex(&hex);
1134                format!("Transaction reverted on-chain (revert_data: {hex})")
1135            }
1136            None => REVERT_REASON_GENERIC.to_string(),
1137        }
1138    }
1139
1140    /// Best-effort recovery of the on-chain revert payload for a transaction entering Failed.
1141    ///
1142    /// Prefers `debug_traceTransaction` with `callTracer`; falls back to an `eth_call` re-executed
1143    /// at the transaction's execution block. Every error is contained and mapped to `None`, so a
1144    /// recovery failure can never propagate out of the status flow. Returns the `0x`-prefixed
1145    /// revert hex when recovered.
1146    async fn recover_revert_data(&self, tx: &TransactionRepoModel) -> Option<String> {
1147        let evm_data = tx.network_data.get_evm_transaction_data().ok()?;
1148        let hash = evm_data.hash.as_ref()?;
1149
1150        // Preferred: debug_traceTransaction. An Err means tracing is unavailable on this RPC,
1151        // so fall through to eth_call.
1152        match self
1153            .provider()
1154            .raw_request_dyn(
1155                "debug_traceTransaction",
1156                json!([hash, {"tracer": "callTracer"}]),
1157            )
1158            .await
1159        {
1160            Ok(trace) => {
1161                if let Some(hex) = extract_trace_output(&trace) {
1162                    info!(
1163                        tx_id = %tx.id,
1164                        method = "trace",
1165                        "recovered on-chain revert data"
1166                    );
1167                    return Some(hex);
1168                }
1169                debug!(
1170                    tx_id = %tx.id,
1171                    "trace returned no revert output; trying eth_call fallback"
1172                );
1173            }
1174            Err(e) => {
1175                warn!(
1176                    tx_id = %tx.id,
1177                    error = %e,
1178                    "debug_traceTransaction unavailable; trying eth_call fallback"
1179                );
1180            }
1181        }
1182
1183        // Fallback: re-fetch the receipt to learn the execution block, then re-run the
1184        // transaction as an eth_call at that block.
1185        let block_number = match self.provider().get_transaction_receipt(hash).await {
1186            Ok(Some(receipt)) => match receipt.block_number {
1187                Some(bn) => bn,
1188                None => {
1189                    debug!(
1190                        tx_id = %tx.id,
1191                        "receipt missing block number; skipping eth_call recovery"
1192                    );
1193                    return None;
1194                }
1195            },
1196            Ok(None) => {
1197                debug!(
1198                    tx_id = %tx.id,
1199                    "no receipt found; skipping eth_call recovery"
1200                );
1201                return None;
1202            }
1203            Err(e) => {
1204                warn!(
1205                    tx_id = %tx.id,
1206                    error = %e,
1207                    "failed to re-fetch receipt for revert recovery"
1208                );
1209                return None;
1210            }
1211        };
1212
1213        let request = match build_revert_call_request(&evm_data) {
1214            Ok(req) => req,
1215            Err(e) => {
1216                warn!(
1217                    tx_id = %tx.id,
1218                    error = %e,
1219                    "failed to reconstruct eth_call request for revert recovery"
1220                );
1221                return None;
1222            }
1223        };
1224
1225        match self
1226            .provider()
1227            .get_call_revert_data(&request, block_number)
1228            .await
1229        {
1230            Ok(Some(bytes)) if !bytes.is_empty() => {
1231                let hex = format!("0x{}", hex::encode(&bytes));
1232                info!(
1233                    tx_id = %tx.id,
1234                    method = "eth_call",
1235                    "recovered on-chain revert data"
1236                );
1237                Some(hex)
1238            }
1239            Ok(_) => {
1240                debug!(tx_id = %tx.id, "no revert data recovered");
1241                None
1242            }
1243            Err(e) => {
1244                warn!(
1245                    tx_id = %tx.id,
1246                    error = %e,
1247                    "eth_call revert recovery failed"
1248                );
1249                None
1250            }
1251        }
1252    }
1253
1254    /// Handle transactions stuck in Sent (prepared but not submitted)
1255    async fn handle_sent_state(
1256        &self,
1257        tx: TransactionRepoModel,
1258    ) -> Result<TransactionRepoModel, TransactionError> {
1259        debug!(
1260            tx_id = %tx.id,
1261            relayer_id = %tx.relayer_id,
1262            "handling Sent state"
1263        );
1264
1265        // Check if transaction should be replaced with NOOP (expired, too many attempts on rollup, etc.)
1266        let (should_noop, reason) = self.should_noop(&tx).await?;
1267        if should_noop {
1268            debug!(
1269                tx_id = %tx.id,
1270                relayer_id = %tx.relayer_id,
1271                "preparing NOOP for sent transaction"
1272            );
1273            let update = self.prepare_noop_update_request(&tx, false, reason).await?;
1274            let updated_tx = self
1275                .transaction_repository()
1276                .partial_update(tx.id.clone(), update)
1277                .await?;
1278
1279            self.send_transaction_submit_job(&updated_tx).await?;
1280            let res = self.send_transaction_update_notification(&updated_tx).await;
1281            if let Err(e) = res {
1282                error!(
1283                    tx_id = %updated_tx.id,
1284                    relayer_id = %updated_tx.relayer_id,
1285                    status = ?updated_tx.status,
1286                    error = %e,
1287                    "sending transaction update notification failed for Sent state NOOP"
1288                );
1289            }
1290            return Ok(updated_tx);
1291        }
1292
1293        // Transaction was prepared but submission job may have failed
1294        // Re-queue a resend job if it's been stuck for a while
1295        let age_since_sent = get_age_since_status_change(&tx)?;
1296
1297        if age_since_sent > get_evm_resend_timeout() {
1298            warn!(
1299                tx_id = %tx.id,
1300                relayer_id = %tx.relayer_id,
1301                age_seconds = age_since_sent.num_seconds(),
1302                "transaction stuck in Sent, queuing resubmit job with repricing"
1303            );
1304
1305            // Queue resubmit job to reprice the transaction for better acceptance
1306            self.send_transaction_resubmit_job(&tx).await?;
1307        }
1308
1309        self.update_transaction_status_if_needed(tx, TransactionStatus::Sent, None)
1310            .await
1311    }
1312
1313    /// Determines if we should attempt hash recovery for a stuck transaction.
1314    ///
1315    /// This is an expensive operation, so we only do it when:
1316    /// - Transaction has been in Submitted status for a while (> 2 minutes)
1317    /// - Transaction has had at least 2 resubmission attempts (hashes.len() > 1)
1318    /// - Haven't tried recovery too recently (to avoid repeated attempts)
1319    fn should_try_hash_recovery(
1320        &self,
1321        tx: &TransactionRepoModel,
1322    ) -> Result<bool, TransactionError> {
1323        // Only try recovery for transactions stuck in Submitted
1324        if tx.status != TransactionStatus::Submitted {
1325            return Ok(false);
1326        }
1327
1328        // Must have multiple hashes (indicating resubmissions happened)
1329        if tx.hashes.len() <= 1 {
1330            return Ok(false);
1331        }
1332
1333        // Only try if transaction has been stuck for a while
1334        let age = get_age_of_sent_at(tx)?;
1335        let min_age_for_recovery = get_evm_min_age_for_hash_recovery();
1336
1337        if age < min_age_for_recovery {
1338            return Ok(false);
1339        }
1340
1341        // Check if we've had enough resubmission attempts (more attempts = more likely to have wrong hash)
1342        // Only try recovery if we have at least 3 hashes (2 resubmissions)
1343        if tx.hashes.len() < EVM_MIN_HASHES_FOR_RECOVERY {
1344            return Ok(false);
1345        }
1346
1347        Ok(true)
1348    }
1349
1350    /// Attempts to recover transaction status by checking all historical hashes.
1351    ///
1352    /// When a transaction is resubmitted multiple times due to timeouts, the database
1353    /// may contain multiple hashes. The "current" hash (network_data.hash) might not
1354    /// be the one that actually got mined. This method checks all historical hashes
1355    /// to find if any were mined, and updates the database with the correct one.
1356    ///
1357    /// Returns the updated transaction model if recovery was successful, None otherwise.
1358    async fn try_recover_with_historical_hashes(
1359        &self,
1360        tx: &TransactionRepoModel,
1361        evm_data: &crate::models::EvmTransactionData,
1362    ) -> Result<Option<TransactionRepoModel>, TransactionError> {
1363        warn!(
1364            tx_id = %tx.id,
1365            relayer_id = %tx.relayer_id,
1366            current_hash = ?evm_data.hash,
1367            total_hashes = %tx.hashes.len(),
1368            "attempting hash recovery - checking historical hashes"
1369        );
1370
1371        // Check each historical hash (most recent first, since it's more likely)
1372        for (idx, historical_hash) in tx.hashes.iter().rev().enumerate() {
1373            // Skip if this is the current hash (already checked)
1374            if Some(historical_hash) == evm_data.hash.as_ref() {
1375                continue;
1376            }
1377
1378            debug!(
1379                tx_id = %tx.id,
1380                relayer_id = %tx.relayer_id,
1381                hash = %historical_hash,
1382                index = %idx,
1383                "checking historical hash"
1384            );
1385
1386            // Try to get receipt for this hash
1387            match self
1388                .provider()
1389                .get_transaction_receipt(historical_hash)
1390                .await
1391            {
1392                Ok(Some(receipt)) => {
1393                    warn!(
1394                        tx_id = %tx.id,
1395                        relayer_id = %tx.relayer_id,
1396                        mined_hash = %historical_hash,
1397                        wrong_hash = ?evm_data.hash,
1398                        block_number = ?receipt.block_number,
1399                        "RECOVERED: found mined transaction with historical hash - correcting database"
1400                    );
1401
1402                    // Update with correct hash and Mined status
1403                    // Let the normal status check flow handle confirmation checking
1404                    let updated_tx = self
1405                        .update_transaction_with_corrected_hash(
1406                            tx,
1407                            evm_data,
1408                            historical_hash,
1409                            TransactionStatus::Mined,
1410                        )
1411                        .await?;
1412
1413                    return Ok(Some(updated_tx));
1414                }
1415                Ok(None) => {
1416                    // This hash not found either, continue to next
1417                    continue;
1418                }
1419                Err(e) => {
1420                    // Network error, log but continue checking other hashes
1421                    warn!(
1422                        tx_id = %tx.id,
1423                        relayer_id = %tx.relayer_id,
1424                        hash = %historical_hash,
1425                        error = %e,
1426                        "error checking historical hash, continuing to next"
1427                    );
1428                    continue;
1429                }
1430            }
1431        }
1432
1433        // None of the historical hashes found on-chain
1434        debug!(
1435            tx_id = %tx.id,
1436            relayer_id = %tx.relayer_id,
1437            "hash recovery completed - no historical hashes found on-chain"
1438        );
1439        Ok(None)
1440    }
1441
1442    /// Updates transaction with the corrected hash and status
1443    ///
1444    /// Returns the updated transaction model and sends a notification about the status change.
1445    async fn update_transaction_with_corrected_hash(
1446        &self,
1447        tx: &TransactionRepoModel,
1448        evm_data: &crate::models::EvmTransactionData,
1449        correct_hash: &str,
1450        status: TransactionStatus,
1451    ) -> Result<TransactionRepoModel, TransactionError> {
1452        let mut corrected_data = evm_data.clone();
1453        corrected_data.hash = Some(correct_hash.to_string());
1454
1455        let updated_tx = self
1456            .transaction_repository()
1457            .partial_update(
1458                tx.id.clone(),
1459                TransactionUpdateRequest {
1460                    network_data: Some(NetworkTransactionData::Evm(corrected_data)),
1461                    status: Some(status),
1462                    ..Default::default()
1463                },
1464            )
1465            .await?;
1466
1467        // Send notification about the recovered transaction
1468        if let Err(e) = self.send_transaction_update_notification(&updated_tx).await {
1469            error!(
1470                tx_id = %updated_tx.id,
1471                relayer_id = %updated_tx.relayer_id,
1472                error = %e,
1473                "failed to send notification for hash recovery"
1474            );
1475        }
1476
1477        Ok(updated_tx)
1478    }
1479}
1480
1481#[cfg(test)]
1482mod tests {
1483    use crate::{
1484        config::{EvmNetworkConfig, NetworkConfigCommon},
1485        domain::transaction::evm::{EvmRelayerTransaction, MockPriceCalculatorTrait},
1486        jobs::MockJobProducerTrait,
1487        models::{
1488            evm::Speed, EvmTransactionData, NetworkConfigData, NetworkRepoModel,
1489            NetworkTransactionData, NetworkType, RelayerEvmPolicy, RelayerNetworkPolicy,
1490            RelayerRepoModel, RpcConfig, TransactionReceipt, TransactionRepoModel,
1491            TransactionStatus, U256,
1492        },
1493        repositories::{
1494            MockNetworkRepository, MockRelayerRepository, MockTransactionCounterTrait,
1495            MockTransactionRepository,
1496        },
1497        services::{provider::MockEvmProviderTrait, signer::MockSigner},
1498    };
1499    use alloy::{
1500        consensus::{Eip658Value, Receipt, ReceiptWithBloom},
1501        network::AnyReceiptEnvelope,
1502        primitives::{b256, Address, BlockHash, Bloom, TxHash},
1503    };
1504    use chrono::{Duration, Utc};
1505    use std::sync::Arc;
1506
1507    /// Helper struct holding all the mocks we often need
1508    pub struct TestMocks {
1509        pub provider: MockEvmProviderTrait,
1510        pub relayer_repo: MockRelayerRepository,
1511        pub network_repo: MockNetworkRepository,
1512        pub tx_repo: MockTransactionRepository,
1513        pub job_producer: MockJobProducerTrait,
1514        pub signer: MockSigner,
1515        pub counter: MockTransactionCounterTrait,
1516        pub price_calc: MockPriceCalculatorTrait,
1517    }
1518
1519    /// Returns a default `TestMocks` with zero-configuration stubs.
1520    /// You can override expectations in each test as needed.
1521    pub fn default_test_mocks() -> TestMocks {
1522        TestMocks {
1523            provider: MockEvmProviderTrait::new(),
1524            relayer_repo: MockRelayerRepository::new(),
1525            network_repo: MockNetworkRepository::new(),
1526            tx_repo: MockTransactionRepository::new(),
1527            job_producer: MockJobProducerTrait::new(),
1528            signer: MockSigner::new(),
1529            counter: MockTransactionCounterTrait::new(),
1530            price_calc: MockPriceCalculatorTrait::new(),
1531        }
1532    }
1533
1534    /// Returns a `TestMocks` with network repository configured for prepare_noop_update_request tests.
1535    pub fn default_test_mocks_with_network() -> TestMocks {
1536        let mut mocks = default_test_mocks();
1537        // Set up default expectation for get_by_chain_id that prepare_noop_update_request tests need
1538        mocks
1539            .network_repo
1540            .expect_get_by_chain_id()
1541            .returning(|network_type, chain_id| {
1542                if network_type == NetworkType::Evm && chain_id == 1 {
1543                    Ok(Some(create_test_network_model()))
1544                } else {
1545                    Ok(None)
1546                }
1547            });
1548        mocks
1549    }
1550
1551    /// Creates a test NetworkRepoModel for chain_id 1 (mainnet)
1552    pub fn create_test_network_model() -> NetworkRepoModel {
1553        let evm_config = EvmNetworkConfig {
1554            common: NetworkConfigCommon {
1555                network: "mainnet".to_string(),
1556                from: None,
1557                rpc_urls: Some(vec![RpcConfig::new("https://rpc.example.com".to_string())]),
1558                explorer_urls: Some(vec!["https://explorer.example.com".to_string()]),
1559                average_blocktime_ms: Some(12000),
1560                is_testnet: Some(false),
1561                tags: Some(vec!["mainnet".to_string()]),
1562            },
1563            chain_id: Some(1),
1564            required_confirmations: Some(12),
1565            features: Some(vec!["eip1559".to_string()]),
1566            symbol: Some("ETH".to_string()),
1567            gas_price_cache: None,
1568        };
1569        NetworkRepoModel {
1570            id: "evm:mainnet".to_string(),
1571            name: "mainnet".to_string(),
1572            network_type: NetworkType::Evm,
1573            config: NetworkConfigData::Evm(evm_config),
1574        }
1575    }
1576
1577    /// Creates a test NetworkRepoModel for chain_id 42161 (Arbitrum-like) with no-mempool tag
1578    pub fn create_test_no_mempool_network_model() -> NetworkRepoModel {
1579        let evm_config = EvmNetworkConfig {
1580            common: NetworkConfigCommon {
1581                network: "arbitrum".to_string(),
1582                from: None,
1583                rpc_urls: Some(vec![crate::models::RpcConfig::new(
1584                    "https://arb-rpc.example.com".to_string(),
1585                )]),
1586                explorer_urls: Some(vec!["https://arb-explorer.example.com".to_string()]),
1587                average_blocktime_ms: Some(1000),
1588                is_testnet: Some(false),
1589                tags: Some(vec![
1590                    "arbitrum".to_string(),
1591                    "rollup".to_string(),
1592                    "no-mempool".to_string(),
1593                ]),
1594            },
1595            chain_id: Some(42161),
1596            required_confirmations: Some(12),
1597            features: Some(vec!["eip1559".to_string()]),
1598            symbol: Some("ETH".to_string()),
1599            gas_price_cache: None,
1600        };
1601        NetworkRepoModel {
1602            id: "evm:arbitrum".to_string(),
1603            name: "arbitrum".to_string(),
1604            network_type: NetworkType::Evm,
1605            config: NetworkConfigData::Evm(evm_config),
1606        }
1607    }
1608
1609    /// Minimal "builder" for TransactionRepoModel.
1610    /// Allows quick creation of a test transaction with default fields,
1611    /// then updates them based on the provided status or overrides.
1612    pub fn make_test_transaction(status: TransactionStatus) -> TransactionRepoModel {
1613        TransactionRepoModel {
1614            id: "test-tx-id".to_string(),
1615            relayer_id: "test-relayer-id".to_string(),
1616            status,
1617            status_reason: None,
1618            created_at: Utc::now().to_rfc3339(),
1619            sent_at: None,
1620            confirmed_at: None,
1621            valid_until: None,
1622            delete_at: None,
1623            network_type: NetworkType::Evm,
1624            network_data: NetworkTransactionData::Evm(EvmTransactionData {
1625                chain_id: 1,
1626                from: "0xSender".to_string(),
1627                to: Some("0xRecipient".to_string()),
1628                value: U256::from(0),
1629                data: Some("0xData".to_string()),
1630                gas_limit: Some(21000),
1631                gas_price: Some(20000000000),
1632                max_fee_per_gas: None,
1633                max_priority_fee_per_gas: None,
1634                nonce: None,
1635                signature: None,
1636                hash: None,
1637                speed: Some(Speed::Fast),
1638                raw: None,
1639            }),
1640            priced_at: None,
1641            hashes: Vec::new(),
1642            noop_count: None,
1643            is_canceled: Some(false),
1644            metadata: None,
1645        }
1646    }
1647
1648    /// Minimal "builder" for EvmRelayerTransaction.
1649    /// Takes mock dependencies as arguments.
1650    pub fn make_test_evm_relayer_transaction(
1651        relayer: RelayerRepoModel,
1652        mocks: TestMocks,
1653    ) -> EvmRelayerTransaction<
1654        MockEvmProviderTrait,
1655        MockRelayerRepository,
1656        MockNetworkRepository,
1657        MockTransactionRepository,
1658        MockJobProducerTrait,
1659        MockSigner,
1660        MockTransactionCounterTrait,
1661        MockPriceCalculatorTrait,
1662    > {
1663        EvmRelayerTransaction::new(
1664            relayer,
1665            mocks.provider,
1666            Arc::new(mocks.relayer_repo),
1667            Arc::new(mocks.network_repo),
1668            Arc::new(mocks.tx_repo),
1669            Arc::new(mocks.counter),
1670            Arc::new(mocks.job_producer),
1671            mocks.price_calc,
1672            mocks.signer,
1673        )
1674        .unwrap()
1675    }
1676
1677    fn create_test_relayer() -> RelayerRepoModel {
1678        RelayerRepoModel {
1679            id: "test-relayer-id".to_string(),
1680            name: "Test Relayer".to_string(),
1681            paused: false,
1682            system_disabled: false,
1683            network: "test_network".to_string(),
1684            network_type: NetworkType::Evm,
1685            policies: RelayerNetworkPolicy::Evm(RelayerEvmPolicy::default()),
1686            signer_id: "test_signer".to_string(),
1687            address: "0x".to_string(),
1688            notification_id: None,
1689            custom_rpc_urls: None,
1690            ..Default::default()
1691        }
1692    }
1693
1694    fn make_mock_receipt(status: bool, block_number: Option<u64>) -> TransactionReceipt {
1695        // Use some placeholder values for minimal completeness
1696        let tx_hash = TxHash::from(b256!(
1697            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
1698        ));
1699        let block_hash = BlockHash::from(b256!(
1700            "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
1701        ));
1702        let from_address = Address::from([0x11; 20]);
1703
1704        TransactionReceipt {
1705            inner: alloy::rpc::types::TransactionReceipt {
1706                inner: AnyReceiptEnvelope {
1707                    inner: ReceiptWithBloom {
1708                        receipt: Receipt {
1709                            status: Eip658Value::Eip658(status), // determines success/fail
1710                            cumulative_gas_used: 0,
1711                            logs: vec![],
1712                        },
1713                        logs_bloom: Bloom::ZERO,
1714                    },
1715                    r#type: 0, // Legacy transaction type
1716                },
1717                transaction_hash: tx_hash,
1718                transaction_index: Some(0),
1719                block_hash: block_number.map(|_| block_hash), // only set if mined
1720                block_number,
1721                gas_used: 21000,
1722                effective_gas_price: 1000,
1723                blob_gas_used: None,
1724                blob_gas_price: None,
1725                from: from_address,
1726                to: None,
1727                contract_address: None,
1728            },
1729            other: Default::default(),
1730        }
1731    }
1732
1733    // Tests for `check_transaction_status`
1734    mod check_transaction_status_tests {
1735        use super::*;
1736
1737        #[tokio::test]
1738        async fn test_not_mined() {
1739            let mut mocks = default_test_mocks();
1740            let relayer = create_test_relayer();
1741            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1742
1743            // Provide a hash so we can check for receipt
1744            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1745                evm_data.hash = Some("0xFakeHash".to_string());
1746            }
1747
1748            // Mock that get_transaction_receipt returns None (not mined)
1749            mocks
1750                .provider
1751                .expect_get_transaction_receipt()
1752                .returning(|_| Box::pin(async { Ok(None) }));
1753
1754            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1755
1756            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1757            assert_eq!(status, TransactionStatus::Submitted);
1758        }
1759
1760        #[tokio::test]
1761        async fn test_mined_but_not_confirmed() {
1762            let mut mocks = default_test_mocks();
1763            let relayer = create_test_relayer();
1764            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1765
1766            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1767                evm_data.hash = Some("0xFakeHash".to_string());
1768            }
1769
1770            // Mock a mined receipt with block_number = 100
1771            mocks
1772                .provider
1773                .expect_get_transaction_receipt()
1774                .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1775
1776            // Mock block_number that hasn't reached the confirmation threshold
1777            mocks
1778                .provider
1779                .expect_get_block_number()
1780                .return_once(|| Box::pin(async { Ok(100) }));
1781
1782            // Mock network repository to return a test network model
1783            mocks
1784                .network_repo
1785                .expect_get_by_chain_id()
1786                .returning(|_, _| Ok(Some(create_test_network_model())));
1787
1788            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1789
1790            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1791            assert_eq!(status, TransactionStatus::Mined);
1792        }
1793
1794        #[tokio::test]
1795        async fn test_confirmed() {
1796            let mut mocks = default_test_mocks();
1797            let relayer = create_test_relayer();
1798            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1799
1800            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1801                evm_data.hash = Some("0xFakeHash".to_string());
1802            }
1803
1804            // Mock a mined receipt with block_number = 100
1805            mocks
1806                .provider
1807                .expect_get_transaction_receipt()
1808                .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1809
1810            // Mock block_number that meets the confirmation threshold
1811            mocks
1812                .provider
1813                .expect_get_block_number()
1814                .return_once(|| Box::pin(async { Ok(113) }));
1815
1816            // Mock network repository to return a test network model
1817            mocks
1818                .network_repo
1819                .expect_get_by_chain_id()
1820                .returning(|_, _| Ok(Some(create_test_network_model())));
1821
1822            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1823
1824            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1825            assert_eq!(status, TransactionStatus::Confirmed);
1826        }
1827
1828        #[tokio::test]
1829        async fn test_failed() {
1830            let mut mocks = default_test_mocks();
1831            let relayer = create_test_relayer();
1832            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1833
1834            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1835                evm_data.hash = Some("0xFakeHash".to_string());
1836            }
1837
1838            // Mock a mined receipt with failure
1839            mocks
1840                .provider
1841                .expect_get_transaction_receipt()
1842                .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
1843
1844            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1845
1846            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1847            assert_eq!(status, TransactionStatus::Failed);
1848        }
1849    }
1850
1851    // Tests for `should_resubmit`
1852    mod should_resubmit_tests {
1853        use super::*;
1854        use crate::models::TransactionError;
1855
1856        #[tokio::test]
1857        async fn test_should_resubmit_true() {
1858            let mut mocks = default_test_mocks();
1859            let relayer = create_test_relayer();
1860
1861            // Set sent_at to 600 seconds ago to force resubmission
1862            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1863            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1864
1865            // Mock network repository to return a regular network model
1866            mocks
1867                .network_repo
1868                .expect_get_by_chain_id()
1869                .returning(|_, _| Ok(Some(create_test_network_model())));
1870
1871            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1872            let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1873            assert!(res, "Transaction should be resubmitted after timeout.");
1874        }
1875
1876        #[tokio::test]
1877        async fn test_should_resubmit_false() {
1878            let mut mocks = default_test_mocks();
1879            let relayer = create_test_relayer();
1880
1881            // Make a transaction with status Submitted but recently sent
1882            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1883            tx.sent_at = Some(Utc::now().to_rfc3339());
1884
1885            // Mock network repository to return a regular network model
1886            mocks
1887                .network_repo
1888                .expect_get_by_chain_id()
1889                .returning(|_, _| Ok(Some(create_test_network_model())));
1890
1891            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1892            let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1893            assert!(!res, "Transaction should not be resubmitted immediately.");
1894        }
1895
1896        #[tokio::test]
1897        async fn test_should_resubmit_true_for_no_mempool_network() {
1898            let mut mocks = default_test_mocks();
1899            let relayer = create_test_relayer();
1900
1901            // Set up a transaction that would normally be resubmitted (sent_at long ago)
1902            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1903            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1904
1905            // Set chain_id to match the no-mempool network
1906            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1907                evm_data.chain_id = 42161; // Arbitrum chain ID
1908            }
1909
1910            // Mock network repository to return a no-mempool network model
1911            mocks
1912                .network_repo
1913                .expect_get_by_chain_id()
1914                .returning(|_, _| Ok(Some(create_test_no_mempool_network_model())));
1915
1916            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1917            let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1918            assert!(
1919                res,
1920                "Transaction should be resubmitted for no-mempool networks."
1921            );
1922        }
1923
1924        #[tokio::test]
1925        async fn test_should_resubmit_network_not_found() {
1926            let mut mocks = default_test_mocks();
1927            let relayer = create_test_relayer();
1928
1929            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1930            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1931
1932            // Mock network repository to return None (network not found)
1933            mocks
1934                .network_repo
1935                .expect_get_by_chain_id()
1936                .returning(|_, _| Ok(None));
1937
1938            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1939            let result = evm_transaction.should_resubmit(&tx).await;
1940
1941            assert!(
1942                result.is_err(),
1943                "should_resubmit should return error when network not found"
1944            );
1945            let error = result.unwrap_err();
1946            match error {
1947                TransactionError::UnexpectedError(msg) => {
1948                    assert!(msg.contains("Network with chain id 1 not found"));
1949                }
1950                _ => panic!("Expected UnexpectedError for network not found"),
1951            }
1952        }
1953
1954        #[tokio::test]
1955        async fn test_should_resubmit_network_conversion_error() {
1956            let mut mocks = default_test_mocks();
1957            let relayer = create_test_relayer();
1958
1959            let mut tx = make_test_transaction(TransactionStatus::Submitted);
1960            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1961
1962            // Create a network model with invalid EVM config (missing chain_id)
1963            let invalid_evm_config = EvmNetworkConfig {
1964                common: NetworkConfigCommon {
1965                    network: "invalid-network".to_string(),
1966                    from: None,
1967                    rpc_urls: Some(vec![crate::models::RpcConfig::new(
1968                        "https://rpc.example.com".to_string(),
1969                    )]),
1970                    explorer_urls: Some(vec!["https://explorer.example.com".to_string()]),
1971                    average_blocktime_ms: Some(12000),
1972                    is_testnet: Some(false),
1973                    tags: Some(vec!["testnet".to_string()]),
1974                },
1975                chain_id: None, // This will cause the conversion to fail
1976                required_confirmations: Some(12),
1977                features: Some(vec!["eip1559".to_string()]),
1978                symbol: Some("ETH".to_string()),
1979                gas_price_cache: None,
1980            };
1981            let invalid_network = NetworkRepoModel {
1982                id: "evm:invalid".to_string(),
1983                name: "invalid-network".to_string(),
1984                network_type: NetworkType::Evm,
1985                config: NetworkConfigData::Evm(invalid_evm_config),
1986            };
1987
1988            // Mock network repository to return the invalid network model
1989            mocks
1990                .network_repo
1991                .expect_get_by_chain_id()
1992                .returning(move |_, _| Ok(Some(invalid_network.clone())));
1993
1994            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1995            let result = evm_transaction.should_resubmit(&tx).await;
1996
1997            assert!(
1998                result.is_err(),
1999                "should_resubmit should return error when network conversion fails"
2000            );
2001            let error = result.unwrap_err();
2002            match error {
2003                TransactionError::UnexpectedError(msg) => {
2004                    assert!(msg.contains("Error converting network model to EvmNetwork"));
2005                }
2006                _ => panic!("Expected UnexpectedError for network conversion failure"),
2007            }
2008        }
2009    }
2010
2011    // Tests for `should_noop`
2012    mod should_noop_tests {
2013        use super::*;
2014
2015        #[tokio::test]
2016        async fn test_expired_transaction_triggers_noop() {
2017            let mut mocks = default_test_mocks();
2018            let relayer = create_test_relayer();
2019
2020            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2021            // Force the transaction to be "expired" by setting valid_until in the past
2022            tx.valid_until = Some((Utc::now() - Duration::seconds(10)).to_rfc3339());
2023
2024            // Mock network repository to return a test network model
2025            mocks
2026                .network_repo
2027                .expect_get_by_chain_id()
2028                .returning(|_, _| Ok(Some(create_test_network_model())));
2029
2030            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2031            let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2032            assert!(res, "Expired transaction should be replaced with a NOOP.");
2033            assert!(
2034                reason.is_some(),
2035                "Reason should be provided for expired transaction"
2036            );
2037            assert!(
2038                reason.unwrap().contains("expired"),
2039                "Reason should mention expiration"
2040            );
2041        }
2042
2043        #[tokio::test]
2044        async fn test_too_many_noop_attempts_returns_false() {
2045            let mocks = default_test_mocks();
2046            let relayer = create_test_relayer();
2047
2048            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2049            tx.noop_count = Some(51); // Max is 50, so this should return false
2050
2051            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2052            let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2053            assert!(
2054                !res,
2055                "Transaction with too many NOOP attempts should not be replaced."
2056            );
2057            assert!(
2058                reason.is_none(),
2059                "Reason should not be provided when should_noop is false"
2060            );
2061        }
2062
2063        #[tokio::test]
2064        async fn test_already_noop_returns_false() {
2065            let mut mocks = default_test_mocks();
2066            let relayer = create_test_relayer();
2067
2068            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2069            // Make it a NOOP by setting to=None and value=0
2070            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2071                evm_data.to = None;
2072                evm_data.value = U256::from(0);
2073            }
2074
2075            mocks
2076                .network_repo
2077                .expect_get_by_chain_id()
2078                .returning(|_, _| Ok(Some(create_test_network_model())));
2079
2080            // Mock get_block_by_number for gas limit validation (won't be called since is_noop returns early, but needed for compilation)
2081            mocks.provider.expect_get_block_by_number().returning(|| {
2082                Box::pin(async {
2083                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2084                    let mut block: Block = Block::default();
2085                    block.header.gas_limit = 30_000_000u64;
2086                    Ok(AnyRpcBlock::from(block))
2087                })
2088            });
2089
2090            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2091            let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2092            assert!(
2093                !res,
2094                "Transaction that is already a NOOP should not be replaced."
2095            );
2096            assert!(
2097                reason.is_none(),
2098                "Reason should not be provided when should_noop is false"
2099            );
2100        }
2101
2102        #[tokio::test]
2103        async fn test_rollup_with_too_many_attempts_triggers_noop() {
2104            let mut mocks = default_test_mocks();
2105            let relayer = create_test_relayer();
2106
2107            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2108            // Set chain_id to Arbitrum (rollup network)
2109            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2110                evm_data.chain_id = 42161; // Arbitrum
2111            }
2112            // Set enough hashes to trigger too_many_attempts (> 50)
2113            tx.hashes = vec!["0xHash1".to_string(); 51];
2114
2115            // Mock network repository to return Arbitrum network
2116            mocks
2117                .network_repo
2118                .expect_get_by_chain_id()
2119                .returning(|_, _| Ok(Some(create_test_no_mempool_network_model())));
2120
2121            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2122            let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2123            assert!(
2124                res,
2125                "Rollup transaction with too many attempts should be replaced with NOOP."
2126            );
2127            assert!(
2128                reason.is_some(),
2129                "Reason should be provided for rollup transaction"
2130            );
2131            assert!(
2132                reason.unwrap().contains("too many attempts"),
2133                "Reason should mention too many attempts"
2134            );
2135        }
2136
2137        #[tokio::test]
2138        async fn test_pending_state_timeout_triggers_noop() {
2139            let mut mocks = default_test_mocks();
2140            let relayer = create_test_relayer();
2141
2142            let mut tx = make_test_transaction(TransactionStatus::Pending);
2143            // Set created_at to 3 minutes ago (> 2 minute timeout)
2144            tx.created_at = (Utc::now() - Duration::minutes(3)).to_rfc3339();
2145
2146            mocks
2147                .network_repo
2148                .expect_get_by_chain_id()
2149                .returning(|_, _| Ok(Some(create_test_network_model())));
2150
2151            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2152            let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2153            assert!(
2154                res,
2155                "Pending transaction stuck for >2 minutes should be replaced with NOOP."
2156            );
2157            assert!(
2158                reason.is_some(),
2159                "Reason should be provided for pending timeout"
2160            );
2161            assert!(
2162                reason.unwrap().contains("Pending state"),
2163                "Reason should mention Pending state"
2164            );
2165        }
2166
2167        #[tokio::test]
2168        async fn test_valid_transaction_returns_false() {
2169            let mut mocks = default_test_mocks();
2170            let relayer = create_test_relayer();
2171
2172            let tx = make_test_transaction(TransactionStatus::Submitted);
2173            // Transaction is recent, not expired, not on rollup, no issues
2174
2175            mocks
2176                .network_repo
2177                .expect_get_by_chain_id()
2178                .returning(|_, _| Ok(Some(create_test_network_model())));
2179
2180            // Mock get_block_by_number for gas limit validation
2181            mocks.provider.expect_get_block_by_number().returning(|| {
2182                Box::pin(async {
2183                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2184                    let mut block: Block = Block::default();
2185                    block.header.gas_limit = 30_000_000u64;
2186                    Ok(AnyRpcBlock::from(block))
2187                })
2188            });
2189
2190            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2191            let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2192            assert!(!res, "Valid transaction should not be replaced with NOOP.");
2193            assert!(
2194                reason.is_none(),
2195                "Reason should not be provided when should_noop is false"
2196            );
2197        }
2198    }
2199
2200    // Tests for `update_transaction_status_if_needed`
2201    mod update_transaction_status_tests {
2202        use super::*;
2203
2204        #[tokio::test]
2205        async fn test_no_update_when_status_is_same() {
2206            // Create mocks, relayer, and a transaction with status Submitted.
2207            let mocks = default_test_mocks();
2208            let relayer = create_test_relayer();
2209            let tx = make_test_transaction(TransactionStatus::Submitted);
2210            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2211
2212            // When new status is the same as current, update_transaction_status_if_needed
2213            // should simply return the original transaction.
2214            let updated_tx = evm_transaction
2215                .update_transaction_status_if_needed(tx.clone(), TransactionStatus::Submitted, None)
2216                .await
2217                .unwrap();
2218            assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2219            assert_eq!(updated_tx.id, tx.id);
2220        }
2221
2222        #[tokio::test]
2223        async fn test_updates_when_status_differs() {
2224            let mut mocks = default_test_mocks();
2225            let relayer = create_test_relayer();
2226            let tx = make_test_transaction(TransactionStatus::Submitted);
2227
2228            // Mock partial_update to return a transaction with new status
2229            mocks
2230                .tx_repo
2231                .expect_partial_update()
2232                .returning(|_, update| {
2233                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2234                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2235                    Ok(updated_tx)
2236                });
2237
2238            // Mock notification job
2239            mocks
2240                .job_producer
2241                .expect_produce_send_notification_job()
2242                .returning(|_, _| Box::pin(async { Ok(()) }));
2243
2244            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2245            let updated_tx = evm_transaction
2246                .update_transaction_status_if_needed(tx.clone(), TransactionStatus::Mined, None)
2247                .await
2248                .unwrap();
2249
2250            assert_eq!(updated_tx.status, TransactionStatus::Mined);
2251        }
2252
2253        #[tokio::test]
2254        async fn test_updates_with_status_reason() {
2255            let mut mocks = default_test_mocks();
2256            let relayer = create_test_relayer();
2257            let tx = make_test_transaction(TransactionStatus::Submitted);
2258
2259            mocks
2260                .tx_repo
2261                .expect_partial_update()
2262                .withf(|_, update| {
2263                    update.status == Some(TransactionStatus::Failed)
2264                        && update.status_reason == Some("Transaction reverted on-chain".to_string())
2265                })
2266                .returning(|_, update| {
2267                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2268                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2269                    updated_tx.status_reason = update.status_reason.clone();
2270                    Ok(updated_tx)
2271                });
2272
2273            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2274            let updated_tx = evm_transaction
2275                .update_transaction_status_if_needed(
2276                    tx.clone(),
2277                    TransactionStatus::Failed,
2278                    Some("Transaction reverted on-chain".to_string()),
2279                )
2280                .await
2281                .unwrap();
2282
2283            assert_eq!(updated_tx.status, TransactionStatus::Failed);
2284            assert_eq!(
2285                updated_tx.status_reason.as_deref(),
2286                Some("Transaction reverted on-chain")
2287            );
2288        }
2289    }
2290
2291    // Tests for `handle_sent_state`
2292    mod handle_sent_state_tests {
2293        use super::*;
2294
2295        #[tokio::test]
2296        async fn test_sent_state_recent_no_resend() {
2297            let mut mocks = default_test_mocks();
2298            let relayer = create_test_relayer();
2299
2300            let mut tx = make_test_transaction(TransactionStatus::Sent);
2301            // Set sent_at to recent (e.g., 10 seconds ago)
2302            tx.sent_at = Some((Utc::now() - Duration::seconds(10)).to_rfc3339());
2303
2304            // Mock network repository to return a test network model for should_noop check
2305            mocks
2306                .network_repo
2307                .expect_get_by_chain_id()
2308                .returning(|_, _| Ok(Some(create_test_network_model())));
2309
2310            // Mock get_block_by_number for gas limit validation in handle_sent_state
2311            mocks.provider.expect_get_block_by_number().returning(|| {
2312                Box::pin(async {
2313                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2314                    let mut block: Block = Block::default();
2315                    block.header.gas_limit = 30_000_000u64;
2316                    Ok(AnyRpcBlock::from(block))
2317                })
2318            });
2319
2320            // Mock status check job scheduling
2321            mocks
2322                .job_producer
2323                .expect_produce_check_transaction_status_job()
2324                .returning(|_, _| Box::pin(async { Ok(()) }));
2325
2326            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2327            let result = evm_transaction.handle_sent_state(tx.clone()).await.unwrap();
2328
2329            assert_eq!(result.status, TransactionStatus::Sent);
2330        }
2331
2332        #[tokio::test]
2333        async fn test_sent_state_stuck_schedules_resubmit() {
2334            let mut mocks = default_test_mocks();
2335            let relayer = create_test_relayer();
2336
2337            let mut tx = make_test_transaction(TransactionStatus::Sent);
2338            // Set sent_at to long ago (> 30 seconds for resend timeout)
2339            tx.sent_at = Some((Utc::now() - Duration::seconds(60)).to_rfc3339());
2340
2341            // Mock network repository to return a test network model for should_noop check
2342            mocks
2343                .network_repo
2344                .expect_get_by_chain_id()
2345                .returning(|_, _| Ok(Some(create_test_network_model())));
2346
2347            // Mock get_block_by_number for gas limit validation in handle_sent_state
2348            mocks.provider.expect_get_block_by_number().returning(|| {
2349                Box::pin(async {
2350                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2351                    let mut block: Block = Block::default();
2352                    block.header.gas_limit = 30_000_000u64;
2353                    Ok(AnyRpcBlock::from(block))
2354                })
2355            });
2356
2357            // Mock resubmit job scheduling
2358            mocks
2359                .job_producer
2360                .expect_produce_submit_transaction_job()
2361                .returning(|_, _| Box::pin(async { Ok(()) }));
2362
2363            // Mock status check job scheduling
2364            mocks
2365                .job_producer
2366                .expect_produce_check_transaction_status_job()
2367                .returning(|_, _| Box::pin(async { Ok(()) }));
2368
2369            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2370            let result = evm_transaction.handle_sent_state(tx.clone()).await.unwrap();
2371
2372            assert_eq!(result.status, TransactionStatus::Sent);
2373        }
2374    }
2375
2376    // Tests for `prepare_noop_update_request`
2377    mod prepare_noop_update_request_tests {
2378        use super::*;
2379
2380        #[tokio::test]
2381        async fn test_noop_request_without_cancellation() {
2382            // Create a transaction with an initial noop_count of 2 and is_canceled set to false.
2383            let mocks = default_test_mocks_with_network();
2384            let relayer = create_test_relayer();
2385            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2386            tx.noop_count = Some(2);
2387            tx.is_canceled = Some(false);
2388
2389            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2390            let update_req = evm_transaction
2391                .prepare_noop_update_request(&tx, false, None)
2392                .await
2393                .unwrap();
2394
2395            // NOOP count should be incremented: 2 becomes 3.
2396            assert_eq!(update_req.noop_count, Some(3));
2397            // When not cancelling, the is_canceled flag should remain as in the original transaction.
2398            assert_eq!(update_req.is_canceled, Some(false));
2399        }
2400
2401        #[tokio::test]
2402        async fn test_noop_request_with_cancellation() {
2403            // Create a transaction with no initial noop_count (None) and is_canceled false.
2404            let mocks = default_test_mocks_with_network();
2405            let relayer = create_test_relayer();
2406            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2407            tx.noop_count = None;
2408            tx.is_canceled = Some(false);
2409
2410            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2411            let update_req = evm_transaction
2412                .prepare_noop_update_request(&tx, true, None)
2413                .await
2414                .unwrap();
2415
2416            // NOOP count should default to 1.
2417            assert_eq!(update_req.noop_count, Some(1));
2418            // When cancelling, the is_canceled flag should be forced to true.
2419            assert_eq!(update_req.is_canceled, Some(true));
2420        }
2421    }
2422
2423    // Tests for `handle_submitted_state`
2424    mod handle_submitted_state_tests {
2425        use super::*;
2426
2427        #[tokio::test]
2428        async fn test_schedules_resubmit_job() {
2429            let mut mocks = default_test_mocks();
2430            let relayer = create_test_relayer();
2431
2432            // Set sent_at far in the past to force resubmission
2433            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2434            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2435
2436            // Mock network repository to return a test network model for should_noop check
2437            mocks
2438                .network_repo
2439                .expect_get_by_chain_id()
2440                .returning(|_, _| Ok(Some(create_test_network_model())));
2441
2442            // Mock get_block_by_number for gas limit validation
2443            mocks.provider.expect_get_block_by_number().returning(|| {
2444                Box::pin(async {
2445                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2446                    let mut block: Block = Block::default();
2447                    block.header.gas_limit = 30_000_000u64;
2448                    Ok(AnyRpcBlock::from(block))
2449                })
2450            });
2451
2452            // On-chain nonce <= tx nonce (no gap), so resubmission proceeds normally
2453            mocks
2454                .provider
2455                .expect_get_transaction_count()
2456                .returning(|_| Box::pin(async { Ok(10) }));
2457
2458            // Expect the resubmit job to be produced
2459            mocks
2460                .job_producer
2461                .expect_produce_submit_transaction_job()
2462                .returning(|_, _| Box::pin(async { Ok(()) }));
2463
2464            // Expect status check to be scheduled
2465            mocks
2466                .job_producer
2467                .expect_produce_check_transaction_status_job()
2468                .returning(|_, _| Box::pin(async { Ok(()) }));
2469
2470            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2471            let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2472
2473            // We remain in "Submitted" after scheduling the resubmit
2474            assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2475        }
2476
2477        /// When tx_nonce > on_chain_nonce and nonce slots are empty, the tx is
2478        /// blocked by a gap. Should schedule nonce health job and skip resubmission.
2479        #[tokio::test]
2480        async fn test_nonce_gap_detected_schedules_health_skips_resubmit() {
2481            let mut mocks = default_test_mocks();
2482            let relayer = create_test_relayer();
2483
2484            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2485            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2486            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
2487                nonce: Some(274),
2488                hash: Some("0xhash".to_string()),
2489                raw: Some(vec![1, 2, 3]),
2490                ..tx.network_data.get_evm_transaction_data().unwrap()
2491            });
2492
2493            // Mock network repository for should_resubmit
2494            mocks
2495                .network_repo
2496                .expect_get_by_chain_id()
2497                .returning(|_, _| Ok(Some(create_test_network_model())));
2498
2499            // On-chain nonce is 269, tx nonce is 274
2500            mocks
2501                .provider
2502                .expect_get_transaction_count()
2503                .returning(|_| Box::pin(async { Ok(269) }));
2504
2505            // Batch nonce scan: nonces 269-273 are all empty → confirmed gap
2506            mocks
2507                .tx_repo
2508                .expect_get_nonce_occupancy()
2509                .withf(|relayer_id, from, to| {
2510                    relayer_id == "test-relayer-id" && *from == 269 && *to == 274
2511                })
2512                .returning(|_, from, to| Ok((from..to).map(|n| (n, None)).collect()));
2513
2514            // Should schedule nonce health job
2515            mocks
2516                .job_producer
2517                .expect_produce_relayer_health_check_job()
2518                .withf(|job, _| {
2519                    job.metadata.as_ref().map_or(false, |m| {
2520                        m.get("health_check_action") == Some(&"nonce_health".to_string())
2521                    })
2522                })
2523                .returning(|_, _| Box::pin(async { Ok(()) }));
2524
2525            // Should NOT call produce_submit_transaction_job (resubmit skipped)
2526            // mockall will panic if unexpected calls are made
2527
2528            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2529            let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2530
2531            assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2532        }
2533
2534        /// When get_transaction_count fails, the gap check should be skipped
2535        /// and resubmission should proceed normally.
2536        #[tokio::test]
2537        async fn test_nonce_gap_check_rpc_failure_proceeds_to_resubmit() {
2538            let mut mocks = default_test_mocks();
2539            let relayer = create_test_relayer();
2540
2541            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2542            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2543
2544            mocks
2545                .network_repo
2546                .expect_get_by_chain_id()
2547                .returning(|_, _| Ok(Some(create_test_network_model())));
2548
2549            mocks.provider.expect_get_block_by_number().returning(|| {
2550                Box::pin(async {
2551                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2552                    let mut block: Block = Block::default();
2553                    block.header.gas_limit = 30_000_000u64;
2554                    Ok(AnyRpcBlock::from(block))
2555                })
2556            });
2557
2558            // RPC fails for nonce check — should be gracefully skipped
2559            mocks
2560                .provider
2561                .expect_get_transaction_count()
2562                .returning(|_| {
2563                    Box::pin(async {
2564                        Err(crate::services::provider::ProviderError::Other(
2565                            "rpc timeout".to_string(),
2566                        ))
2567                    })
2568                });
2569
2570            // Resubmission should still proceed
2571            mocks
2572                .job_producer
2573                .expect_produce_submit_transaction_job()
2574                .returning(|_, _| Box::pin(async { Ok(()) }));
2575
2576            mocks
2577                .job_producer
2578                .expect_produce_check_transaction_status_job()
2579                .returning(|_, _| Box::pin(async { Ok(()) }));
2580
2581            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2582            let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2583
2584            assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2585        }
2586
2587        /// When all nonce slots between on-chain and tx_nonce are filled by active
2588        /// transactions, no gap exists — resubmission proceeds normally.
2589        #[tokio::test]
2590        async fn test_no_gap_when_slots_filled_proceeds_to_resubmit() {
2591            let mut mocks = default_test_mocks();
2592            let relayer = create_test_relayer();
2593
2594            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2595            tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2596            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
2597                nonce: Some(270),
2598                hash: Some("0xhash".to_string()),
2599                raw: Some(vec![1, 2, 3]),
2600                ..tx.network_data.get_evm_transaction_data().unwrap()
2601            });
2602
2603            mocks
2604                .network_repo
2605                .expect_get_by_chain_id()
2606                .returning(|_, _| Ok(Some(create_test_network_model())));
2607
2608            mocks.provider.expect_get_block_by_number().returning(|| {
2609                Box::pin(async {
2610                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2611                    let mut block: Block = Block::default();
2612                    block.header.gas_limit = 30_000_000u64;
2613                    Ok(AnyRpcBlock::from(block))
2614                })
2615            });
2616
2617            // tx_nonce=270, on_chain=269 → 1 slot to check
2618            mocks
2619                .provider
2620                .expect_get_transaction_count()
2621                .returning(|_| Box::pin(async { Ok(269) }));
2622
2623            // Nonce 269 has an active Submitted tx → no gap
2624            mocks
2625                .tx_repo
2626                .expect_get_nonce_occupancy()
2627                .returning(|_, from, to| {
2628                    Ok((from..to)
2629                        .map(|n| (n, Some(TransactionStatus::Submitted)))
2630                        .collect())
2631                });
2632
2633            // Should proceed to resubmit (no health job expected)
2634            mocks
2635                .job_producer
2636                .expect_produce_submit_transaction_job()
2637                .returning(|_, _| Box::pin(async { Ok(()) }));
2638
2639            mocks
2640                .job_producer
2641                .expect_produce_check_transaction_status_job()
2642                .returning(|_, _| Box::pin(async { Ok(()) }));
2643
2644            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2645            let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2646
2647            assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2648        }
2649    }
2650
2651    // Tests for `handle_pending_state`
2652    mod handle_pending_state_tests {
2653        use super::*;
2654
2655        #[tokio::test]
2656        async fn test_pending_state_no_noop() {
2657            // Create a pending transaction that is fresh (created now).
2658            let mut mocks = default_test_mocks();
2659            let relayer = create_test_relayer();
2660            let mut tx = make_test_transaction(TransactionStatus::Pending);
2661            tx.created_at = Utc::now().to_rfc3339(); // less than one minute old
2662
2663            // Mock network repository to return a test network model
2664            mocks
2665                .network_repo
2666                .expect_get_by_chain_id()
2667                .returning(|_, _| Ok(Some(create_test_network_model())));
2668
2669            // Mock get_block_by_number for gas limit validation
2670            mocks.provider.expect_get_block_by_number().returning(|| {
2671                Box::pin(async {
2672                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2673                    let mut block: Block = Block::default();
2674                    block.header.gas_limit = 30_000_000u64;
2675                    Ok(AnyRpcBlock::from(block))
2676                })
2677            });
2678
2679            // Expect status check to be scheduled when not doing NOOP
2680            mocks
2681                .job_producer
2682                .expect_produce_check_transaction_status_job()
2683                .returning(|_, _| Box::pin(async { Ok(()) }));
2684
2685            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2686            let result = evm_transaction
2687                .handle_pending_state(tx.clone())
2688                .await
2689                .unwrap();
2690
2691            // When should_noop returns false the original transaction is returned unchanged.
2692            assert_eq!(result.id, tx.id);
2693            assert_eq!(result.status, tx.status);
2694            assert_eq!(result.noop_count, tx.noop_count);
2695        }
2696
2697        #[tokio::test]
2698        async fn test_pending_state_with_noop() {
2699            // Create a pending transaction that is old (created 2 minutes ago)
2700            let mut mocks = default_test_mocks();
2701            let relayer = create_test_relayer();
2702            let mut tx = make_test_transaction(TransactionStatus::Pending);
2703            tx.created_at = (Utc::now() - Duration::minutes(2)).to_rfc3339();
2704
2705            // Mock network repository to return a test network model
2706            mocks
2707                .network_repo
2708                .expect_get_by_chain_id()
2709                .returning(|_, _| Ok(Some(create_test_network_model())));
2710
2711            // Mock get_block_by_number for gas limit validation
2712            mocks.provider.expect_get_block_by_number().returning(|| {
2713                Box::pin(async {
2714                    use alloy::{network::AnyRpcBlock, rpc::types::Block};
2715                    let mut block: Block = Block::default();
2716                    block.header.gas_limit = 30_000_000u64;
2717                    Ok(AnyRpcBlock::from(block))
2718                })
2719            });
2720
2721            // Expect partial_update to be called and simulate a Failed update
2722            // (Pending state transactions are marked as Failed, not NOOP, since nonces aren't assigned)
2723            let tx_clone = tx.clone();
2724            mocks
2725                .tx_repo
2726                .expect_partial_update()
2727                .withf(move |id, update| {
2728                    id == "test-tx-id"
2729                        && update.status == Some(TransactionStatus::Failed)
2730                        && update.status_reason.is_some()
2731                })
2732                .returning(move |_, update| {
2733                    let mut updated_tx = tx_clone.clone();
2734                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2735                    updated_tx.status_reason = update.status_reason.clone();
2736                    Ok(updated_tx)
2737                });
2738            // Expect that a notification is produced (no submit job needed for Failed status)
2739            mocks
2740                .job_producer
2741                .expect_produce_send_notification_job()
2742                .returning(|_, _| Box::pin(async { Ok(()) }));
2743
2744            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2745            let result = evm_transaction
2746                .handle_pending_state(tx.clone())
2747                .await
2748                .unwrap();
2749
2750            // Since should_noop returns true for pending timeout, transaction should be marked as Failed
2751            assert_eq!(result.status, TransactionStatus::Failed);
2752            assert!(result.status_reason.is_some());
2753            assert!(result.status_reason.unwrap().contains("Pending state"));
2754        }
2755    }
2756
2757    // Tests for `handle_mined_state`
2758    mod handle_mined_state_tests {
2759        use super::*;
2760
2761        #[tokio::test]
2762        async fn test_updates_status_and_schedules_check() {
2763            let mut mocks = default_test_mocks();
2764            let relayer = create_test_relayer();
2765            // Create a transaction in Submitted state (the mined branch is reached via status check).
2766            let tx = make_test_transaction(TransactionStatus::Submitted);
2767
2768            // Expect schedule_status_check to be called with delay 5.
2769            mocks
2770                .job_producer
2771                .expect_produce_check_transaction_status_job()
2772                .returning(|_, _| Box::pin(async { Ok(()) }));
2773            // Expect partial_update to update the transaction status to Mined.
2774            mocks
2775                .tx_repo
2776                .expect_partial_update()
2777                .returning(|_, update| {
2778                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2779                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2780                    Ok(updated_tx)
2781                });
2782
2783            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2784            let result = evm_transaction
2785                .handle_mined_state(tx.clone())
2786                .await
2787                .unwrap();
2788            assert_eq!(result.status, TransactionStatus::Mined);
2789        }
2790    }
2791
2792    // Tests for `handle_final_state`
2793    mod handle_final_state_tests {
2794        use super::*;
2795
2796        #[tokio::test]
2797        async fn test_final_state_confirmed() {
2798            let mut mocks = default_test_mocks();
2799            let relayer = create_test_relayer();
2800            let tx = make_test_transaction(TransactionStatus::Submitted);
2801
2802            // Expect partial_update to update status to Confirmed.
2803            mocks
2804                .tx_repo
2805                .expect_partial_update()
2806                .returning(|_, update| {
2807                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2808                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2809                    Ok(updated_tx)
2810                });
2811
2812            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2813            let result = evm_transaction
2814                .handle_final_state(tx.clone(), TransactionStatus::Confirmed, None)
2815                .await
2816                .unwrap();
2817            assert_eq!(result.status, TransactionStatus::Confirmed);
2818        }
2819
2820        #[tokio::test]
2821        async fn test_final_state_failed() {
2822            let mut mocks = default_test_mocks();
2823            let relayer = create_test_relayer();
2824            let tx = make_test_transaction(TransactionStatus::Submitted);
2825
2826            // Expect partial_update to update status to Failed with status_reason.
2827            mocks
2828                .tx_repo
2829                .expect_partial_update()
2830                .returning(|_, update| {
2831                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2832                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2833                    updated_tx.status_reason = update.status_reason.clone();
2834                    Ok(updated_tx)
2835                });
2836
2837            let reason = "Transaction reverted on-chain (receipt status: failed)".to_string();
2838            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2839            let result = evm_transaction
2840                .handle_final_state(tx.clone(), TransactionStatus::Failed, Some(reason.clone()))
2841                .await
2842                .unwrap();
2843            assert_eq!(result.status, TransactionStatus::Failed);
2844            assert_eq!(result.status_reason.as_deref(), Some(reason.as_str()));
2845        }
2846
2847        #[tokio::test]
2848        async fn test_final_state_expired() {
2849            let mut mocks = default_test_mocks();
2850            let relayer = create_test_relayer();
2851            let tx = make_test_transaction(TransactionStatus::Submitted);
2852
2853            // Expect partial_update to update status to Expired.
2854            mocks
2855                .tx_repo
2856                .expect_partial_update()
2857                .returning(|_, update| {
2858                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2859                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2860                    Ok(updated_tx)
2861                });
2862
2863            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2864            let result = evm_transaction
2865                .handle_final_state(tx.clone(), TransactionStatus::Expired, None)
2866                .await
2867                .unwrap();
2868            assert_eq!(result.status, TransactionStatus::Expired);
2869        }
2870    }
2871
2872    // Integration tests for `handle_status_impl`
2873    mod handle_status_impl_tests {
2874        use super::*;
2875
2876        #[tokio::test]
2877        async fn test_impl_submitted_branch() {
2878            let mut mocks = default_test_mocks();
2879            let relayer = create_test_relayer();
2880            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2881            tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
2882            // Set a dummy hash so check_transaction_status can proceed.
2883            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2884                evm_data.hash = Some("0xFakeHash".to_string());
2885            }
2886            // Simulate no receipt found.
2887            mocks
2888                .provider
2889                .expect_get_transaction_receipt()
2890                .returning(|_| Box::pin(async { Ok(None) }));
2891            // Mock network repository for should_resubmit check
2892            mocks
2893                .network_repo
2894                .expect_get_by_chain_id()
2895                .returning(|_, _| Ok(Some(create_test_network_model())));
2896            // Expect that a status check job is scheduled.
2897            mocks
2898                .job_producer
2899                .expect_produce_check_transaction_status_job()
2900                .returning(|_, _| Box::pin(async { Ok(()) }));
2901            // Expect update_transaction_status_if_needed to update status to Submitted.
2902            mocks
2903                .tx_repo
2904                .expect_partial_update()
2905                .returning(|_, update| {
2906                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2907                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2908                    Ok(updated_tx)
2909                });
2910
2911            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2912            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2913            assert_eq!(result.status, TransactionStatus::Submitted);
2914        }
2915
2916        #[tokio::test]
2917        async fn test_impl_mined_branch() {
2918            let mut mocks = default_test_mocks();
2919            let relayer = create_test_relayer();
2920            let mut tx = make_test_transaction(TransactionStatus::Submitted);
2921            // Set created_at to be old enough to pass is_too_early_to_resubmit
2922            tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
2923            // Set a dummy hash.
2924            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2925                evm_data.hash = Some("0xFakeHash".to_string());
2926            }
2927            // Simulate a receipt with a block number of 100 and a successful receipt.
2928            mocks
2929                .provider
2930                .expect_get_transaction_receipt()
2931                .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
2932            // Simulate that the current block number is 100 (so confirmations are insufficient).
2933            mocks
2934                .provider
2935                .expect_get_block_number()
2936                .return_once(|| Box::pin(async { Ok(100) }));
2937            // Mock network repository to return a test network model
2938            mocks
2939                .network_repo
2940                .expect_get_by_chain_id()
2941                .returning(|_, _| Ok(Some(create_test_network_model())));
2942            // Mock the notification job that gets sent after status update
2943            mocks
2944                .job_producer
2945                .expect_produce_send_notification_job()
2946                .returning(|_, _| Box::pin(async { Ok(()) }));
2947            // Expect get_by_id to reload the transaction after status change
2948            mocks.tx_repo.expect_get_by_id().returning(|_| {
2949                let updated_tx = make_test_transaction(TransactionStatus::Mined);
2950                Ok(updated_tx)
2951            });
2952            // Expect update_transaction_status_if_needed to update status to Mined.
2953            mocks
2954                .tx_repo
2955                .expect_partial_update()
2956                .returning(|_, update| {
2957                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2958                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2959                    Ok(updated_tx)
2960                });
2961
2962            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2963            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2964            assert_eq!(result.status, TransactionStatus::Mined);
2965        }
2966
2967        #[tokio::test]
2968        async fn test_impl_final_confirmed_branch() {
2969            let mut mocks = default_test_mocks();
2970            let relayer = create_test_relayer();
2971            // Create a transaction with status Confirmed.
2972            let tx = make_test_transaction(TransactionStatus::Confirmed);
2973
2974            // In this branch, check_transaction_status returns the final status immediately,
2975            // so we expect partial_update to update the transaction status to Confirmed.
2976            mocks
2977                .tx_repo
2978                .expect_partial_update()
2979                .returning(|_, update| {
2980                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2981                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
2982                    Ok(updated_tx)
2983                });
2984
2985            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2986            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2987            assert_eq!(result.status, TransactionStatus::Confirmed);
2988        }
2989
2990        #[tokio::test]
2991        async fn test_impl_final_failed_branch() {
2992            let mut mocks = default_test_mocks();
2993            let relayer = create_test_relayer();
2994            // Create a transaction with status Failed.
2995            let tx = make_test_transaction(TransactionStatus::Failed);
2996
2997            mocks
2998                .tx_repo
2999                .expect_partial_update()
3000                .returning(|_, update| {
3001                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3002                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3003                    Ok(updated_tx)
3004                });
3005
3006            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3007            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3008            assert_eq!(result.status, TransactionStatus::Failed);
3009        }
3010
3011        /// Verifies that a Submitted transaction with a failed on-chain receipt
3012        /// transitions to Failed status with a descriptive status_reason.
3013        #[tokio::test]
3014        async fn test_impl_submitted_to_failed_sets_status_reason() {
3015            let mut mocks = default_test_mocks();
3016            let relayer = create_test_relayer();
3017            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3018            tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
3019            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3020                evm_data.hash = Some("0xFakeHash".to_string());
3021            }
3022
3023            // Simulate a receipt with status=false (reverted on-chain).
3024            mocks
3025                .provider
3026                .expect_get_transaction_receipt()
3027                .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
3028
3029            // Revert-data recovery runs on the transition to Failed. The trace path returns no
3030            // usable output here, and the placeholder tx data can't be reconstructed into an
3031            // eth_call, so recovery yields the legacy generic reason.
3032            mocks
3033                .provider
3034                .expect_raw_request_dyn()
3035                .returning(|_, _| Box::pin(async { Ok(serde_json::json!({})) }));
3036
3037            // Mock get_by_id for the DB reload after status change.
3038            let tx_clone = tx.clone();
3039            mocks.tx_repo.expect_get_by_id().returning(move |_| {
3040                let mut reloaded = tx_clone.clone();
3041                reloaded.status = TransactionStatus::Submitted;
3042                Ok(reloaded)
3043            });
3044
3045            // Expect partial_update with status=Failed and a status_reason.
3046            mocks
3047                .tx_repo
3048                .expect_partial_update()
3049                .withf(|_, update| {
3050                    update.status == Some(TransactionStatus::Failed)
3051                        && update.status_reason.is_some()
3052                })
3053                .returning(|_, update| {
3054                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3055                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3056                    updated_tx.status_reason = update.status_reason.clone();
3057                    Ok(updated_tx)
3058                });
3059
3060            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3061            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3062            assert_eq!(result.status, TransactionStatus::Failed);
3063            assert!(result.status_reason.is_some());
3064            assert!(
3065                result
3066                    .status_reason
3067                    .as_ref()
3068                    .unwrap()
3069                    .contains("reverted on-chain"),
3070                "Expected on-chain revert reason, got: {:?}",
3071                result.status_reason
3072            );
3073        }
3074
3075        #[tokio::test]
3076        async fn test_impl_final_expired_branch() {
3077            let mut mocks = default_test_mocks();
3078            let relayer = create_test_relayer();
3079            // Create a transaction with status Expired.
3080            let tx = make_test_transaction(TransactionStatus::Expired);
3081
3082            mocks
3083                .tx_repo
3084                .expect_partial_update()
3085                .returning(|_, update| {
3086                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3087                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3088                    Ok(updated_tx)
3089                });
3090
3091            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3092            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3093            assert_eq!(result.status, TransactionStatus::Expired);
3094        }
3095    }
3096
3097    // Tests for on-chain revert-data recovery on the transition into Failed.
3098    mod revert_data_recovery_tests {
3099        use super::*;
3100        use crate::services::provider::ProviderError;
3101        use alloy::primitives::Bytes;
3102
3103        const LEGACY_REASON: &str = "Transaction reverted on-chain (receipt status: failed)";
3104
3105        /// A Submitted tx (aged past the resubmit grace) with valid EVM fields and a hash,
3106        /// poised to transition to Failed from a failed receipt.
3107        fn failing_submitted_tx() -> TransactionRepoModel {
3108            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3109            tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
3110            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3111                evm_data.from = "0x0000000000000000000000000000000000000001".to_string();
3112                evm_data.to = Some("0x0000000000000000000000000000000000000002".to_string());
3113                evm_data.data = Some("0xabcdef".to_string());
3114                evm_data.hash = Some("0xFakeHash".to_string());
3115            }
3116            tx
3117        }
3118
3119        /// Receipt with status=false (reverted) at block 100. Reused for the status check and
3120        /// the recovery re-fetch.
3121        fn expect_failed_receipt(mocks: &mut TestMocks) {
3122            mocks
3123                .provider
3124                .expect_get_transaction_receipt()
3125                .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
3126        }
3127
3128        /// DB reload after the status change, plus a partial_update that captures the
3129        /// status_reason onto the returned transaction so assertions can inspect it.
3130        fn expect_reload_and_capture(mocks: &mut TestMocks) {
3131            mocks.tx_repo.expect_get_by_id().returning(|_| {
3132                let mut reloaded = failing_submitted_tx();
3133                reloaded.status = TransactionStatus::Submitted;
3134                Ok(reloaded)
3135            });
3136            mocks
3137                .tx_repo
3138                .expect_partial_update()
3139                .returning(|_, update| {
3140                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3141                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3142                    updated_tx.status_reason = update.status_reason.clone();
3143                    Ok(updated_tx)
3144                });
3145        }
3146
3147        // eth_call surfaces revert data -> enriched status_reason.
3148        #[tokio::test]
3149        async fn test_eth_call_revert_data_enriches_status_reason() {
3150            let mut mocks = default_test_mocks();
3151            let relayer = create_test_relayer();
3152            let tx = failing_submitted_tx();
3153
3154            expect_failed_receipt(&mut mocks);
3155            // Trace path yields no output -> fall back to eth_call.
3156            mocks
3157                .provider
3158                .expect_raw_request_dyn()
3159                .returning(|_, _| Box::pin(async { Ok(serde_json::json!({})) }));
3160            mocks
3161                .provider
3162                .expect_get_call_revert_data()
3163                .returning(|_, _| {
3164                    Box::pin(async { Ok(Some(Bytes::from(hex::decode("5592f1b2").unwrap()))) })
3165                });
3166            expect_reload_and_capture(&mut mocks);
3167
3168            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3169            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3170            assert_eq!(result.status, TransactionStatus::Failed);
3171            let reason = result.status_reason.unwrap();
3172            assert!(
3173                reason.contains("revert_data: 0x5592f1b2"),
3174                "expected enriched revert_data, got: {reason}"
3175            );
3176        }
3177
3178        // No revert data recovered -> generic reason, byte-for-byte.
3179        #[tokio::test]
3180        async fn test_no_revert_data_uses_legacy_reason() {
3181            let mut mocks = default_test_mocks();
3182            let relayer = create_test_relayer();
3183            let tx = failing_submitted_tx();
3184
3185            expect_failed_receipt(&mut mocks);
3186            mocks
3187                .provider
3188                .expect_raw_request_dyn()
3189                .returning(|_, _| Box::pin(async { Ok(serde_json::json!({})) }));
3190            mocks
3191                .provider
3192                .expect_get_call_revert_data()
3193                .returning(|_, _| Box::pin(async { Ok(None) }));
3194            expect_reload_and_capture(&mut mocks);
3195
3196            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3197            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3198            assert_eq!(result.status, TransactionStatus::Failed);
3199            assert_eq!(result.status_reason.unwrap(), LEGACY_REASON);
3200        }
3201
3202        // Tx already Failed -> no recovery RPC, reason not rewritten.
3203        #[tokio::test]
3204        async fn test_already_failed_skips_recovery() {
3205            // No provider expectations set: any recovery RPC (trace/eth_call/receipt) would panic.
3206            let mocks = default_test_mocks();
3207            let relayer = create_test_relayer();
3208            let tx = make_test_transaction(TransactionStatus::Failed);
3209
3210            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3211            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3212            assert_eq!(result.status, TransactionStatus::Failed);
3213            assert!(
3214                result.status_reason.is_none(),
3215                "status_reason should not be rewritten on re-poll, got: {:?}",
3216                result.status_reason
3217            );
3218        }
3219
3220        // Trace output is used and eth_call is NOT invoked.
3221        #[tokio::test]
3222        async fn test_trace_output_used_without_eth_call() {
3223            let mut mocks = default_test_mocks();
3224            let relayer = create_test_relayer();
3225            let tx = failing_submitted_tx();
3226
3227            expect_failed_receipt(&mut mocks);
3228            mocks.provider.expect_raw_request_dyn().returning(|_, _| {
3229                Box::pin(async { Ok(serde_json::json!({"output": "0xdeadbeef"})) })
3230            });
3231            // eth_call fallback must never run when the trace path produced a payload.
3232            mocks.provider.expect_get_call_revert_data().never();
3233            expect_reload_and_capture(&mut mocks);
3234
3235            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3236            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3237            assert_eq!(result.status, TransactionStatus::Failed);
3238            assert!(result
3239                .status_reason
3240                .unwrap()
3241                .contains("revert_data: 0xdeadbeef"));
3242        }
3243
3244        // Trace unavailable -> eth_call fallback produces the payload.
3245        #[tokio::test]
3246        async fn test_trace_unavailable_falls_back_to_eth_call() {
3247            let mut mocks = default_test_mocks();
3248            let relayer = create_test_relayer();
3249            let tx = failing_submitted_tx();
3250
3251            expect_failed_receipt(&mut mocks);
3252            mocks.provider.expect_raw_request_dyn().returning(|_, _| {
3253                Box::pin(async { Err(ProviderError::Other("method not supported".to_string())) })
3254            });
3255            mocks
3256                .provider
3257                .expect_get_call_revert_data()
3258                // The reconstructed call must carry the persisted fee context (legacy gas_price
3259                // here) so contracts that branch on tx.gasprice revert as they did on-chain.
3260                .withf(|req, _block| req.gas_price == Some(20000000000))
3261                .returning(|_, _| {
3262                    Box::pin(async { Ok(Some(Bytes::from(hex::decode("08c379a0").unwrap()))) })
3263                });
3264            expect_reload_and_capture(&mut mocks);
3265
3266            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3267            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3268            assert_eq!(result.status, TransactionStatus::Failed);
3269            assert!(result
3270                .status_reason
3271                .unwrap()
3272                .contains("revert_data: 0x08c379a0"));
3273        }
3274
3275        // Both trace and eth_call fail/empty -> generic reason, status still Failed, no error.
3276        #[tokio::test]
3277        async fn test_both_paths_fail_uses_legacy_reason() {
3278            let mut mocks = default_test_mocks();
3279            let relayer = create_test_relayer();
3280            let tx = failing_submitted_tx();
3281
3282            expect_failed_receipt(&mut mocks);
3283            mocks.provider.expect_raw_request_dyn().returning(|_, _| {
3284                Box::pin(async { Err(ProviderError::Other("trace down".to_string())) })
3285            });
3286            mocks
3287                .provider
3288                .expect_get_call_revert_data()
3289                .returning(|_, _| {
3290                    Box::pin(async { Err(ProviderError::Other("eth_call down".to_string())) })
3291                });
3292            expect_reload_and_capture(&mut mocks);
3293
3294            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3295            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3296            assert_eq!(result.status, TransactionStatus::Failed);
3297            assert_eq!(result.status_reason.unwrap(), LEGACY_REASON);
3298        }
3299
3300        /// Builds a relayer whose EVM policy sets `include_revert_data` to the given value.
3301        fn relayer_with_revert_policy(include: Option<bool>) -> RelayerRepoModel {
3302            let mut relayer = create_test_relayer();
3303            relayer.policies = RelayerNetworkPolicy::Evm(RelayerEvmPolicy {
3304                include_revert_data: include,
3305                ..Default::default()
3306            });
3307            relayer
3308        }
3309
3310        // include_revert_data = Some(false) -> zero recovery RPCs, generic reason.
3311        #[tokio::test]
3312        async fn test_recovery_disabled_issues_no_recovery_rpcs() {
3313            let mut mocks = default_test_mocks();
3314            let relayer = relayer_with_revert_policy(Some(false));
3315            let tx = failing_submitted_tx();
3316
3317            // Only the status-check receipt fetch is allowed; recovery must issue none of these.
3318            expect_failed_receipt(&mut mocks);
3319            mocks.provider.expect_raw_request_dyn().never();
3320            mocks.provider.expect_get_call_revert_data().never();
3321            expect_reload_and_capture(&mut mocks);
3322
3323            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3324            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3325            assert_eq!(result.status, TransactionStatus::Failed);
3326            assert_eq!(result.status_reason.unwrap(), LEGACY_REASON);
3327        }
3328
3329        // None (default-on) and Some(true) -> recovery is attempted.
3330        #[tokio::test]
3331        async fn test_recovery_attempted_when_enabled() {
3332            for include in [None, Some(true)] {
3333                let mut mocks = default_test_mocks();
3334                let relayer = relayer_with_revert_policy(include);
3335                let tx = failing_submitted_tx();
3336
3337                expect_failed_receipt(&mut mocks);
3338                // Asserting the trace RPC is issued exactly once proves recovery was attempted.
3339                mocks
3340                    .provider
3341                    .expect_raw_request_dyn()
3342                    .times(1)
3343                    .returning(|_, _| {
3344                        Box::pin(async { Ok(serde_json::json!({"output": "0xfeed"})) })
3345                    });
3346                expect_reload_and_capture(&mut mocks);
3347
3348                let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3349                let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3350                assert_eq!(result.status, TransactionStatus::Failed);
3351                assert!(
3352                    result
3353                        .status_reason
3354                        .unwrap()
3355                        .contains("revert_data: 0xfeed"),
3356                    "recovery should run for include_revert_data = {include:?}"
3357                );
3358            }
3359        }
3360    }
3361
3362    // Tests for circuit breaker functionality
3363    mod circuit_breaker_tests {
3364        use super::*;
3365        use crate::jobs::StatusCheckContext;
3366
3367        /// Helper to create a context that should trigger the circuit breaker
3368        fn create_triggered_context() -> StatusCheckContext {
3369            StatusCheckContext::new(
3370                30, // consecutive_failures: exceeds EVM threshold of 25
3371                50, // total_failures
3372                60, // total_retries
3373                25, // max_consecutive_failures (EVM default)
3374                75, // max_total_failures (EVM default)
3375                NetworkType::Evm,
3376            )
3377        }
3378
3379        /// Helper to create a context that should NOT trigger the circuit breaker
3380        fn create_safe_context() -> StatusCheckContext {
3381            StatusCheckContext::new(
3382                5,  // consecutive_failures: below threshold
3383                10, // total_failures
3384                15, // total_retries
3385                25, // max_consecutive_failures
3386                75, // max_total_failures
3387                NetworkType::Evm,
3388            )
3389        }
3390
3391        /// Helper to create a context that triggers via total failures (safety net)
3392        fn create_total_triggered_context() -> StatusCheckContext {
3393            StatusCheckContext::new(
3394                5,   // consecutive_failures: below threshold
3395                80,  // total_failures: exceeds EVM threshold of 75
3396                100, // total_retries
3397                25,  // max_consecutive_failures
3398                75,  // max_total_failures
3399                NetworkType::Evm,
3400            )
3401        }
3402
3403        #[tokio::test]
3404        async fn test_circuit_breaker_pending_marks_as_failed() {
3405            let mut mocks = default_test_mocks();
3406            let relayer = create_test_relayer();
3407            let tx = make_test_transaction(TransactionStatus::Pending);
3408
3409            // Expect partial_update to be called with Failed status
3410            mocks
3411                .tx_repo
3412                .expect_partial_update()
3413                .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3414                .returning(|_, update| {
3415                    let mut updated_tx = make_test_transaction(TransactionStatus::Pending);
3416                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3417                    updated_tx.status_reason = update.status_reason.clone();
3418                    Ok(updated_tx)
3419                });
3420
3421            // Mock notification (best effort, may or may not be called)
3422            mocks
3423                .job_producer
3424                .expect_produce_send_notification_job()
3425                .returning(|_, _| Box::pin(async { Ok(()) }));
3426
3427            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3428            let ctx = create_triggered_context();
3429
3430            let result = evm_transaction
3431                .handle_status_impl(tx, Some(ctx))
3432                .await
3433                .unwrap();
3434
3435            assert_eq!(result.status, TransactionStatus::Failed);
3436            assert!(result.status_reason.is_some());
3437            assert!(result.status_reason.unwrap().contains("consecutive errors"));
3438        }
3439
3440        #[tokio::test]
3441        async fn test_circuit_breaker_sent_marks_as_failed() {
3442            let mut mocks = default_test_mocks();
3443            let relayer = create_test_relayer();
3444            let tx = make_test_transaction(TransactionStatus::Sent);
3445
3446            // Expect partial_update to be called with Failed status
3447            mocks
3448                .tx_repo
3449                .expect_partial_update()
3450                .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3451                .returning(|_, update| {
3452                    let mut updated_tx = make_test_transaction(TransactionStatus::Sent);
3453                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3454                    updated_tx.status_reason = update.status_reason.clone();
3455                    Ok(updated_tx)
3456                });
3457
3458            // Mock notification
3459            mocks
3460                .job_producer
3461                .expect_produce_send_notification_job()
3462                .returning(|_, _| Box::pin(async { Ok(()) }));
3463
3464            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3465            let ctx = create_triggered_context();
3466
3467            let result = evm_transaction
3468                .handle_status_impl(tx, Some(ctx))
3469                .await
3470                .unwrap();
3471
3472            assert_eq!(result.status, TransactionStatus::Failed);
3473        }
3474
3475        #[tokio::test]
3476        async fn test_circuit_breaker_submitted_triggers_noop() {
3477            let mut mocks = default_test_mocks();
3478            let relayer = create_test_relayer();
3479            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3480            tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3481
3482            // Mock network repository for NOOP processing
3483            mocks
3484                .network_repo
3485                .expect_get_by_chain_id()
3486                .returning(|_, _| Ok(Some(create_test_network_model())));
3487
3488            // Expect partial_update to be called with NOOP indicator
3489            mocks
3490                .tx_repo
3491                .expect_partial_update()
3492                .returning(|_, update| {
3493                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3494                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3495                    updated_tx.status_reason = update.status_reason.clone();
3496                    updated_tx.noop_count = update.noop_count;
3497                    Ok(updated_tx)
3498                });
3499
3500            // Mock resubmit job (NOOP triggers resubmit)
3501            mocks
3502                .job_producer
3503                .expect_produce_submit_transaction_job()
3504                .returning(|_, _| Box::pin(async { Ok(()) }));
3505
3506            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3507            let ctx = create_triggered_context();
3508
3509            let result = evm_transaction
3510                .handle_status_impl(tx, Some(ctx))
3511                .await
3512                .unwrap();
3513
3514            // NOOP processing should succeed
3515            assert!(result.noop_count.is_some());
3516        }
3517
3518        #[tokio::test]
3519        async fn test_circuit_breaker_noop_tx_excluded() {
3520            let mut mocks = default_test_mocks();
3521            let relayer = create_test_relayer();
3522
3523            // Create a NOOP transaction (to: self, value: 0, data: "0x")
3524            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3525            tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3526            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3527                evm_data.to = Some(evm_data.from.clone()); // to == from (NOOP indicator)
3528                evm_data.value = U256::from(0);
3529                evm_data.data = Some("0x".to_string());
3530                evm_data.hash = Some("0xFakeHash".to_string());
3531            }
3532
3533            // NOOP transactions should NOT trigger circuit breaker
3534            // Instead, they should go through normal status checking
3535            mocks
3536                .provider
3537                .expect_get_transaction_receipt()
3538                .returning(|_| Box::pin(async { Ok(None) }));
3539
3540            mocks
3541                .network_repo
3542                .expect_get_by_chain_id()
3543                .returning(|_, _| Ok(Some(create_test_network_model())));
3544
3545            mocks
3546                .job_producer
3547                .expect_produce_check_transaction_status_job()
3548                .returning(|_, _| Box::pin(async { Ok(()) }));
3549
3550            // Mock resubmit job (may be triggered by normal status flow for stuck transactions)
3551            mocks
3552                .job_producer
3553                .expect_produce_submit_transaction_job()
3554                .returning(|_, _| Box::pin(async { Ok(()) }));
3555
3556            mocks
3557                .tx_repo
3558                .expect_partial_update()
3559                .returning(|_, update| {
3560                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3561                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3562                    Ok(updated_tx)
3563                });
3564
3565            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3566            let ctx = create_triggered_context();
3567
3568            let result = evm_transaction
3569                .handle_status_impl(tx, Some(ctx))
3570                .await
3571                .unwrap();
3572
3573            // NOOP tx should continue normal processing, not be force-failed
3574            assert_eq!(result.status, TransactionStatus::Submitted);
3575        }
3576
3577        #[tokio::test]
3578        async fn test_circuit_breaker_total_failures_triggers() {
3579            let mut mocks = default_test_mocks();
3580            let relayer = create_test_relayer();
3581            let tx = make_test_transaction(TransactionStatus::Pending);
3582
3583            // Expect partial_update to be called with Failed status
3584            mocks
3585                .tx_repo
3586                .expect_partial_update()
3587                .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3588                .returning(|_, update| {
3589                    let mut updated_tx = make_test_transaction(TransactionStatus::Pending);
3590                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3591                    updated_tx.status_reason = update.status_reason.clone();
3592                    Ok(updated_tx)
3593                });
3594
3595            mocks
3596                .job_producer
3597                .expect_produce_send_notification_job()
3598                .returning(|_, _| Box::pin(async { Ok(()) }));
3599
3600            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3601            // Use context that triggers via total failures (safety net)
3602            let ctx = create_total_triggered_context();
3603
3604            let result = evm_transaction
3605                .handle_status_impl(tx, Some(ctx))
3606                .await
3607                .unwrap();
3608
3609            assert_eq!(result.status, TransactionStatus::Failed);
3610            assert!(result.status_reason.is_some());
3611        }
3612
3613        #[tokio::test]
3614        async fn test_circuit_breaker_below_threshold_continues_normally() {
3615            let mut mocks = default_test_mocks();
3616            let relayer = create_test_relayer();
3617            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3618            tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3619
3620            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3621                evm_data.hash = Some("0xFakeHash".to_string());
3622            }
3623
3624            // Below threshold, should continue with normal status checking
3625            mocks
3626                .provider
3627                .expect_get_transaction_receipt()
3628                .returning(|_| Box::pin(async { Ok(None) }));
3629
3630            mocks
3631                .network_repo
3632                .expect_get_by_chain_id()
3633                .returning(|_, _| Ok(Some(create_test_network_model())));
3634
3635            mocks
3636                .job_producer
3637                .expect_produce_check_transaction_status_job()
3638                .returning(|_, _| Box::pin(async { Ok(()) }));
3639
3640            mocks
3641                .tx_repo
3642                .expect_partial_update()
3643                .returning(|_, update| {
3644                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3645                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3646                    Ok(updated_tx)
3647                });
3648
3649            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3650            let ctx = create_safe_context();
3651
3652            let result = evm_transaction
3653                .handle_status_impl(tx, Some(ctx))
3654                .await
3655                .unwrap();
3656
3657            // Should continue normal processing, not trigger circuit breaker
3658            assert_eq!(result.status, TransactionStatus::Submitted);
3659        }
3660
3661        #[tokio::test]
3662        async fn test_circuit_breaker_no_context_continues_normally() {
3663            let mut mocks = default_test_mocks();
3664            let relayer = create_test_relayer();
3665            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3666            tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3667
3668            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3669                evm_data.hash = Some("0xFakeHash".to_string());
3670            }
3671
3672            // No context means no circuit breaker, should continue normally
3673            mocks
3674                .provider
3675                .expect_get_transaction_receipt()
3676                .returning(|_| Box::pin(async { Ok(None) }));
3677
3678            mocks
3679                .network_repo
3680                .expect_get_by_chain_id()
3681                .returning(|_, _| Ok(Some(create_test_network_model())));
3682
3683            mocks
3684                .job_producer
3685                .expect_produce_check_transaction_status_job()
3686                .returning(|_, _| Box::pin(async { Ok(()) }));
3687
3688            mocks
3689                .tx_repo
3690                .expect_partial_update()
3691                .returning(|_, update| {
3692                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3693                    updated_tx.status = update.status.unwrap_or(updated_tx.status);
3694                    Ok(updated_tx)
3695                });
3696
3697            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3698
3699            // Pass None for context
3700            let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3701
3702            // Should continue normal processing
3703            assert_eq!(result.status, TransactionStatus::Submitted);
3704        }
3705
3706        #[tokio::test]
3707        async fn test_circuit_breaker_final_state_early_return() {
3708            let mocks = default_test_mocks();
3709            let relayer = create_test_relayer();
3710            // Transaction is already in final state
3711            let tx = make_test_transaction(TransactionStatus::Confirmed);
3712
3713            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3714            let ctx = create_triggered_context();
3715
3716            // Even with triggered context, final states should return early
3717            let result = evm_transaction
3718                .handle_status_impl(tx, Some(ctx))
3719                .await
3720                .unwrap();
3721
3722            assert_eq!(result.status, TransactionStatus::Confirmed);
3723        }
3724    }
3725
3726    // Tests for hash recovery functions
3727    mod hash_recovery_tests {
3728        use super::*;
3729
3730        #[tokio::test]
3731        async fn test_should_try_hash_recovery_not_submitted() {
3732            let mocks = default_test_mocks();
3733            let relayer = create_test_relayer();
3734
3735            let mut tx = make_test_transaction(TransactionStatus::Sent);
3736            tx.hashes = vec![
3737                "0xHash1".to_string(),
3738                "0xHash2".to_string(),
3739                "0xHash3".to_string(),
3740            ];
3741
3742            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3743            let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3744
3745            assert!(
3746                !result,
3747                "Should not attempt recovery for non-Submitted transactions"
3748            );
3749        }
3750
3751        #[tokio::test]
3752        async fn test_should_try_hash_recovery_not_enough_hashes() {
3753            let mocks = default_test_mocks();
3754            let relayer = create_test_relayer();
3755
3756            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3757            tx.hashes = vec!["0xHash1".to_string()]; // Only 1 hash
3758            tx.sent_at = Some((Utc::now() - Duration::minutes(3)).to_rfc3339());
3759
3760            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3761            let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3762
3763            assert!(
3764                !result,
3765                "Should not attempt recovery with insufficient hashes"
3766            );
3767        }
3768
3769        #[tokio::test]
3770        async fn test_should_try_hash_recovery_too_recent() {
3771            let mocks = default_test_mocks();
3772            let relayer = create_test_relayer();
3773
3774            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3775            tx.hashes = vec![
3776                "0xHash1".to_string(),
3777                "0xHash2".to_string(),
3778                "0xHash3".to_string(),
3779            ];
3780            tx.sent_at = Some(Utc::now().to_rfc3339()); // Recent
3781
3782            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3783            let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3784
3785            assert!(
3786                !result,
3787                "Should not attempt recovery for recently sent transactions"
3788            );
3789        }
3790
3791        #[tokio::test]
3792        async fn test_should_try_hash_recovery_success() {
3793            let mocks = default_test_mocks();
3794            let relayer = create_test_relayer();
3795
3796            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3797            tx.hashes = vec![
3798                "0xHash1".to_string(),
3799                "0xHash2".to_string(),
3800                "0xHash3".to_string(),
3801            ];
3802            tx.sent_at = Some((Utc::now() - Duration::minutes(3)).to_rfc3339());
3803
3804            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3805            let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3806
3807            assert!(
3808                result,
3809                "Should attempt recovery for stuck transactions with multiple hashes"
3810            );
3811        }
3812
3813        #[tokio::test]
3814        async fn test_try_recover_no_historical_hash_found() {
3815            let mut mocks = default_test_mocks();
3816            let relayer = create_test_relayer();
3817
3818            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3819            tx.hashes = vec![
3820                "0xHash1".to_string(),
3821                "0xHash2".to_string(),
3822                "0xHash3".to_string(),
3823            ];
3824
3825            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3826                evm_data.hash = Some("0xHash3".to_string());
3827            }
3828
3829            // Mock provider to return None for all hash lookups
3830            mocks
3831                .provider
3832                .expect_get_transaction_receipt()
3833                .returning(|_| Box::pin(async { Ok(None) }));
3834
3835            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3836            let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3837            let result = evm_transaction
3838                .try_recover_with_historical_hashes(&tx, &evm_data)
3839                .await
3840                .unwrap();
3841
3842            assert!(
3843                result.is_none(),
3844                "Should return None when no historical hash is found"
3845            );
3846        }
3847
3848        #[tokio::test]
3849        async fn test_try_recover_finds_mined_historical_hash() {
3850            let mut mocks = default_test_mocks();
3851            let relayer = create_test_relayer();
3852
3853            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3854            tx.hashes = vec![
3855                "0xHash1".to_string(),
3856                "0xHash2".to_string(), // This one is mined
3857                "0xHash3".to_string(),
3858            ];
3859
3860            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3861                evm_data.hash = Some("0xHash3".to_string()); // Current hash (wrong one)
3862            }
3863
3864            // Mock provider to return None for Hash1 and Hash3, but receipt for Hash2
3865            mocks
3866                .provider
3867                .expect_get_transaction_receipt()
3868                .returning(|hash| {
3869                    if hash == "0xHash2" {
3870                        Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) })
3871                    } else {
3872                        Box::pin(async { Ok(None) })
3873                    }
3874                });
3875
3876            // Mock partial_update for correcting the hash
3877            let tx_clone = tx.clone();
3878            mocks
3879                .tx_repo
3880                .expect_partial_update()
3881                .returning(move |_, update| {
3882                    let mut updated_tx = tx_clone.clone();
3883                    if let Some(status) = update.status {
3884                        updated_tx.status = status;
3885                    }
3886                    if let Some(NetworkTransactionData::Evm(ref evm_data)) = update.network_data {
3887                        if let NetworkTransactionData::Evm(ref mut updated_evm) =
3888                            updated_tx.network_data
3889                        {
3890                            updated_evm.hash = evm_data.hash.clone();
3891                        }
3892                    }
3893                    Ok(updated_tx)
3894                });
3895
3896            // Mock notification job
3897            mocks
3898                .job_producer
3899                .expect_produce_send_notification_job()
3900                .returning(|_, _| Box::pin(async { Ok(()) }));
3901
3902            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3903            let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3904            let result = evm_transaction
3905                .try_recover_with_historical_hashes(&tx, &evm_data)
3906                .await
3907                .unwrap();
3908
3909            assert!(result.is_some(), "Should recover the transaction");
3910            let recovered_tx = result.unwrap();
3911            assert_eq!(recovered_tx.status, TransactionStatus::Mined);
3912        }
3913
3914        #[tokio::test]
3915        async fn test_try_recover_network_error_continues() {
3916            let mut mocks = default_test_mocks();
3917            let relayer = create_test_relayer();
3918
3919            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3920            tx.hashes = vec![
3921                "0xHash1".to_string(),
3922                "0xHash2".to_string(), // Network error
3923                "0xHash3".to_string(), // This one is mined
3924            ];
3925
3926            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3927                evm_data.hash = Some("0xHash1".to_string());
3928            }
3929
3930            // Mock provider to return error for Hash2, receipt for Hash3
3931            mocks
3932                .provider
3933                .expect_get_transaction_receipt()
3934                .returning(|hash| {
3935                    if hash == "0xHash2" {
3936                        Box::pin(async { Err(crate::services::provider::ProviderError::Timeout) })
3937                    } else if hash == "0xHash3" {
3938                        Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) })
3939                    } else {
3940                        Box::pin(async { Ok(None) })
3941                    }
3942                });
3943
3944            // Mock partial_update for correcting the hash
3945            let tx_clone = tx.clone();
3946            mocks
3947                .tx_repo
3948                .expect_partial_update()
3949                .returning(move |_, update| {
3950                    let mut updated_tx = tx_clone.clone();
3951                    if let Some(status) = update.status {
3952                        updated_tx.status = status;
3953                    }
3954                    Ok(updated_tx)
3955                });
3956
3957            // Mock notification job
3958            mocks
3959                .job_producer
3960                .expect_produce_send_notification_job()
3961                .returning(|_, _| Box::pin(async { Ok(()) }));
3962
3963            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3964            let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3965            let result = evm_transaction
3966                .try_recover_with_historical_hashes(&tx, &evm_data)
3967                .await
3968                .unwrap();
3969
3970            assert!(
3971                result.is_some(),
3972                "Should continue checking after network error and find mined hash"
3973            );
3974        }
3975
3976        #[tokio::test]
3977        async fn test_update_transaction_with_corrected_hash() {
3978            let mut mocks = default_test_mocks();
3979            let relayer = create_test_relayer();
3980
3981            let mut tx = make_test_transaction(TransactionStatus::Submitted);
3982            if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3983                evm_data.hash = Some("0xWrongHash".to_string());
3984            }
3985
3986            // Mock partial_update
3987            mocks
3988                .tx_repo
3989                .expect_partial_update()
3990                .returning(move |_, update| {
3991                    let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3992                    if let Some(status) = update.status {
3993                        updated_tx.status = status;
3994                    }
3995                    if let Some(NetworkTransactionData::Evm(ref evm_data)) = update.network_data {
3996                        if let NetworkTransactionData::Evm(ref mut updated_evm) =
3997                            updated_tx.network_data
3998                        {
3999                            updated_evm.hash = evm_data.hash.clone();
4000                        }
4001                    }
4002                    Ok(updated_tx)
4003                });
4004
4005            // Mock notification job
4006            mocks
4007                .job_producer
4008                .expect_produce_send_notification_job()
4009                .returning(|_, _| Box::pin(async { Ok(()) }));
4010
4011            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4012            let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
4013            let result = evm_transaction
4014                .update_transaction_with_corrected_hash(
4015                    &tx,
4016                    &evm_data,
4017                    "0xCorrectHash",
4018                    TransactionStatus::Mined,
4019                )
4020                .await
4021                .unwrap();
4022
4023            assert_eq!(result.status, TransactionStatus::Mined);
4024            if let NetworkTransactionData::Evm(ref updated_evm) = result.network_data {
4025                assert_eq!(updated_evm.hash.as_ref().unwrap(), "0xCorrectHash");
4026            }
4027        }
4028    }
4029
4030    // Tests for check_transaction_status edge cases
4031    mod check_transaction_status_edge_cases {
4032        use super::*;
4033
4034        #[tokio::test]
4035        async fn test_missing_hash_returns_error() {
4036            let mocks = default_test_mocks();
4037            let relayer = create_test_relayer();
4038
4039            let tx = make_test_transaction(TransactionStatus::Submitted);
4040            // Hash is None by default
4041
4042            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4043            let result = evm_transaction.check_transaction_status(&tx).await;
4044
4045            assert!(result.is_err(), "Should return error when hash is missing");
4046        }
4047
4048        #[tokio::test]
4049        async fn test_pending_status_early_return() {
4050            let mocks = default_test_mocks();
4051            let relayer = create_test_relayer();
4052
4053            let tx = make_test_transaction(TransactionStatus::Pending);
4054
4055            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4056            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
4057
4058            assert_eq!(
4059                status,
4060                TransactionStatus::Pending,
4061                "Should return Pending without querying blockchain"
4062            );
4063        }
4064
4065        #[tokio::test]
4066        async fn test_sent_status_early_return() {
4067            let mocks = default_test_mocks();
4068            let relayer = create_test_relayer();
4069
4070            let tx = make_test_transaction(TransactionStatus::Sent);
4071
4072            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4073            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
4074
4075            assert_eq!(
4076                status,
4077                TransactionStatus::Sent,
4078                "Should return Sent without querying blockchain"
4079            );
4080        }
4081
4082        #[tokio::test]
4083        async fn test_final_state_early_return() {
4084            let mocks = default_test_mocks();
4085            let relayer = create_test_relayer();
4086
4087            let tx = make_test_transaction(TransactionStatus::Confirmed);
4088
4089            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4090            let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
4091
4092            assert_eq!(
4093                status,
4094                TransactionStatus::Confirmed,
4095                "Should return final state without querying blockchain"
4096            );
4097        }
4098    }
4099
4100    mod nonce_recovery_tests {
4101        use super::*;
4102        use crate::domain::transaction::evm::evm_transaction::TX_NONCE_RECONCILE_TRIGGER;
4103        use crate::jobs::StatusCheckContext;
4104
4105        /// Test reconcile_tx_nonce_state with on_chain_nonce > tx_nonce → marks Failed
4106        #[tokio::test]
4107        async fn test_nonce_recovery_nonce_consumed_externally() {
4108            let mut mocks = default_test_mocks();
4109            let relayer = create_test_relayer();
4110
4111            let mut tx = make_test_transaction(TransactionStatus::Submitted);
4112            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4113                nonce: Some(5),
4114                hash: Some("0xhash".to_string()),
4115                raw: Some(vec![1, 2, 3]),
4116                ..tx.network_data.get_evm_transaction_data().unwrap()
4117            });
4118            tx.sent_at = Some(Utc::now().to_rfc3339());
4119
4120            // No receipt for current hash
4121            mocks
4122                .provider
4123                .expect_get_transaction_receipt()
4124                .returning(|_| Box::pin(async { Ok(None) }));
4125
4126            // On-chain nonce is 10, tx nonce is 5 → consumed externally
4127            mocks
4128                .provider
4129                .expect_get_transaction_count()
4130                .returning(|_| Box::pin(async { Ok(10) }));
4131
4132            // Should update to Failed status
4133            let tx_clone = tx.clone();
4134            mocks
4135                .tx_repo
4136                .expect_partial_update()
4137                .withf(|_, update| {
4138                    update.status == Some(TransactionStatus::Failed)
4139                        && update
4140                            .status_reason
4141                            .as_ref()
4142                            .map(|r| r.contains("consumed externally"))
4143                            .unwrap_or(false)
4144                })
4145                .returning(move |_, update| {
4146                    let mut updated_tx = tx_clone.clone();
4147                    updated_tx.status = update.status.unwrap();
4148                    updated_tx.status_reason = update.status_reason.clone();
4149                    Ok(updated_tx)
4150                });
4151
4152            // Should schedule nonce health job after detecting external consumption
4153            mocks
4154                .job_producer
4155                .expect_produce_relayer_health_check_job()
4156                .withf(|job, scheduled_on| {
4157                    job.relayer_id == "test-relayer-id"
4158                        && job.metadata.as_ref().map_or(false, |m| {
4159                            m.get("health_check_action") == Some(&"nonce_health".to_string())
4160                        })
4161                        && scheduled_on.is_none()
4162                })
4163                .returning(|_, _| Box::pin(async { Ok(()) }));
4164
4165            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4166            let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
4167
4168            assert!(result.is_ok());
4169            let recovered = result.unwrap();
4170            assert!(recovered.is_some(), "Expected Some(tx) for consumed nonce");
4171            assert_eq!(recovered.unwrap().status, TransactionStatus::Failed);
4172        }
4173
4174        /// Test reconcile_tx_nonce_state with on_chain_nonce <= tx_nonce → returns None
4175        #[tokio::test]
4176        async fn test_nonce_recovery_nonce_not_consumed() {
4177            let mut mocks = default_test_mocks();
4178            let relayer = create_test_relayer();
4179
4180            let mut tx = make_test_transaction(TransactionStatus::Submitted);
4181            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4182                nonce: Some(5),
4183                hash: Some("0xhash".to_string()),
4184                raw: Some(vec![1, 2, 3]),
4185                ..tx.network_data.get_evm_transaction_data().unwrap()
4186            });
4187
4188            // No receipt for current hash
4189            mocks
4190                .provider
4191                .expect_get_transaction_receipt()
4192                .returning(|_| Box::pin(async { Ok(None) }));
4193
4194            // On-chain nonce is 5, same as tx nonce → not consumed yet
4195            mocks
4196                .provider
4197                .expect_get_transaction_count()
4198                .returning(|_| Box::pin(async { Ok(5) }));
4199
4200            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4201            let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
4202
4203            assert!(result.is_ok());
4204            assert!(
4205                result.unwrap().is_none(),
4206                "Expected None when nonce not consumed"
4207            );
4208        }
4209
4210        /// Test reconcile_tx_nonce_state with receipt found → returns None (defer to normal flow)
4211        #[tokio::test]
4212        async fn test_nonce_recovery_receipt_found() {
4213            let mut mocks = default_test_mocks();
4214            let relayer = create_test_relayer();
4215
4216            let mut tx = make_test_transaction(TransactionStatus::Submitted);
4217            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4218                nonce: Some(5),
4219                hash: Some("0xhash".to_string()),
4220                raw: Some(vec![1, 2, 3]),
4221                ..tx.network_data.get_evm_transaction_data().unwrap()
4222            });
4223
4224            // Receipt exists for current hash — defer to normal flow
4225            mocks
4226                .provider
4227                .expect_get_transaction_receipt()
4228                .returning(|_| {
4229                    let receipt = make_mock_receipt(true, Some(100));
4230                    Box::pin(async move { Ok(Some(receipt)) })
4231                });
4232
4233            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4234            let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
4235
4236            assert!(result.is_ok());
4237            assert!(
4238                result.unwrap().is_none(),
4239                "Expected None when receipt found — defer to normal flow"
4240            );
4241        }
4242
4243        /// Test reconcile_tx_nonce_state with RPC errors during receipt check → returns None
4244        /// Must NOT proceed to force-fail via nonce comparison when hash checks were incomplete
4245        #[tokio::test]
4246        async fn test_nonce_recovery_rpc_error_prevents_force_fail() {
4247            let mut mocks = default_test_mocks();
4248            let relayer = create_test_relayer();
4249
4250            let mut tx = make_test_transaction(TransactionStatus::Submitted);
4251            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4252                nonce: Some(5),
4253                hash: Some("0xhash".to_string()),
4254                raw: Some(vec![1, 2, 3]),
4255                ..tx.network_data.get_evm_transaction_data().unwrap()
4256            });
4257
4258            // Receipt check FAILS with RPC error
4259            mocks
4260                .provider
4261                .expect_get_transaction_receipt()
4262                .returning(|_| {
4263                    Box::pin(async {
4264                        Err(crate::services::provider::ProviderError::Other(
4265                            "RPC timeout".to_string(),
4266                        ))
4267                    })
4268                });
4269
4270            // get_transaction_count should NOT be called — we bail before reaching it
4271            // (no expectation set = will panic if called)
4272
4273            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4274            let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
4275
4276            assert!(result.is_ok());
4277            assert!(
4278                result.unwrap().is_none(),
4279                "Expected None when RPC errors occurred — must not force-fail on incomplete data"
4280            );
4281        }
4282
4283        /// Test handle_status_impl with nonce_error_hint metadata triggers recovery
4284        #[tokio::test]
4285        async fn test_handle_status_impl_nonce_recovery_hint() {
4286            let mut mocks = default_test_mocks();
4287            let relayer = create_test_relayer();
4288
4289            let mut tx = make_test_transaction(TransactionStatus::Submitted);
4290            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4291                nonce: Some(5),
4292                hash: Some("0xhash".to_string()),
4293                raw: Some(vec![1, 2, 3]),
4294                ..tx.network_data.get_evm_transaction_data().unwrap()
4295            });
4296            tx.sent_at = Some(Utc::now().to_rfc3339());
4297
4298            // No receipt for current hash
4299            mocks
4300                .provider
4301                .expect_get_transaction_receipt()
4302                .returning(|_| Box::pin(async { Ok(None) }));
4303
4304            // On-chain nonce > tx nonce → consumed externally
4305            mocks
4306                .provider
4307                .expect_get_transaction_count()
4308                .returning(|_| Box::pin(async { Ok(10) }));
4309
4310            // Should update to Failed
4311            let tx_clone = tx.clone();
4312            mocks
4313                .tx_repo
4314                .expect_partial_update()
4315                .returning(move |_, update| {
4316                    let mut updated_tx = tx_clone.clone();
4317                    if let Some(status) = update.status {
4318                        updated_tx.status = status;
4319                    }
4320                    updated_tx.status_reason = update.status_reason.clone();
4321                    Ok(updated_tx)
4322                });
4323
4324            // Should schedule nonce health job after detecting external consumption
4325            mocks
4326                .job_producer
4327                .expect_produce_relayer_health_check_job()
4328                .returning(|_, _| Box::pin(async { Ok(()) }));
4329
4330            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4331
4332            // Build context with nonce_error_hint metadata
4333            let mut metadata = std::collections::HashMap::new();
4334            metadata.insert(
4335                TX_NONCE_RECONCILE_TRIGGER.to_string(),
4336                "NonceTooLow".to_string(),
4337            );
4338            let context = StatusCheckContext::default().with_job_metadata(Some(metadata));
4339
4340            let result = evm_transaction.handle_status_impl(tx, Some(context)).await;
4341            assert!(result.is_ok());
4342            assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4343        }
4344    }
4345
4346    mod circuit_breaker_sent_state_tests {
4347        use super::*;
4348        use crate::jobs::StatusCheckContext;
4349
4350        /// Test circuit breaker on Sent tx with nonce → issues NOOP + submit job
4351        #[tokio::test]
4352        async fn test_circuit_breaker_sent_with_nonce_issues_noop() {
4353            let mut mocks = default_test_mocks_with_network();
4354            let relayer = create_test_relayer();
4355
4356            let mut tx = make_test_transaction(TransactionStatus::Sent);
4357            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4358                nonce: Some(5),
4359                hash: Some("0xhash".to_string()),
4360                raw: Some(vec![1, 2, 3]),
4361                ..tx.network_data.get_evm_transaction_data().unwrap()
4362            });
4363            tx.sent_at = Some(Utc::now().to_rfc3339());
4364
4365            // process_noop_transaction calls prepare_noop_update_request → partial_update
4366            let tx_clone = tx.clone();
4367            mocks
4368                .tx_repo
4369                .expect_partial_update()
4370                .returning(move |_, update| {
4371                    let mut updated_tx = tx_clone.clone();
4372                    if let Some(status) = update.status {
4373                        updated_tx.status = status;
4374                    }
4375                    updated_tx.status_reason = update.status_reason.clone();
4376                    Ok(updated_tx)
4377                });
4378
4379            // Should produce a submit job (for the NOOP)
4380            mocks
4381                .job_producer
4382                .expect_produce_submit_transaction_job()
4383                .times(1)
4384                .returning(|_, _| Box::pin(async { Ok(()) }));
4385
4386            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4387
4388            // Circuit breaker context that should trigger
4389            let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4390
4391            let result = evm_transaction
4392                .handle_circuit_breaker_safely(tx, &ctx)
4393                .await;
4394            assert!(result.is_ok());
4395        }
4396
4397        /// Test circuit breaker on Sent tx without nonce → marks Failed
4398        #[tokio::test]
4399        async fn test_circuit_breaker_sent_without_nonce_marks_failed() {
4400            let mut mocks = default_test_mocks();
4401            let relayer = create_test_relayer();
4402
4403            let mut tx = make_test_transaction(TransactionStatus::Sent);
4404            // No nonce assigned
4405            tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4406                nonce: None,
4407                hash: None,
4408                raw: None,
4409                ..tx.network_data.get_evm_transaction_data().unwrap()
4410            });
4411            tx.sent_at = Some(Utc::now().to_rfc3339());
4412
4413            // Should mark as Failed
4414            let tx_clone = tx.clone();
4415            mocks
4416                .tx_repo
4417                .expect_partial_update()
4418                .withf(|_, update| update.status == Some(TransactionStatus::Failed))
4419                .returning(move |_, update| {
4420                    let mut updated_tx = tx_clone.clone();
4421                    updated_tx.status = update.status.unwrap();
4422                    Ok(updated_tx)
4423                });
4424
4425            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4426
4427            let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4428
4429            let result = evm_transaction
4430                .handle_circuit_breaker_safely(tx, &ctx)
4431                .await;
4432            assert!(result.is_ok());
4433            assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4434        }
4435
4436        /// Test circuit breaker on Pending tx → marks Failed (unchanged behavior)
4437        #[tokio::test]
4438        async fn test_circuit_breaker_pending_marks_failed() {
4439            let mut mocks = default_test_mocks();
4440            let relayer = create_test_relayer();
4441
4442            let tx = make_test_transaction(TransactionStatus::Pending);
4443
4444            // Should mark as Failed
4445            let tx_clone = tx.clone();
4446            mocks
4447                .tx_repo
4448                .expect_partial_update()
4449                .withf(|_, update| update.status == Some(TransactionStatus::Failed))
4450                .returning(move |_, update| {
4451                    let mut updated_tx = tx_clone.clone();
4452                    updated_tx.status = update.status.unwrap();
4453                    Ok(updated_tx)
4454                });
4455
4456            let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4457
4458            let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4459
4460            let result = evm_transaction
4461                .handle_circuit_breaker_safely(tx, &ctx)
4462                .await;
4463            assert!(result.is_ok());
4464            assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4465        }
4466    }
4467}