1use alloy::network::ReceiptResponse;
6use alloy::primitives::{Address, TxKind};
7use alloy::rpc::types::{TransactionInput, TransactionRequest};
8use chrono::{DateTime, Duration, Utc};
9use eyre::Result;
10use serde_json::json;
11use tracing::{debug, error, info, warn};
12
13use super::super::common::is_active_nonce_status;
14use super::EvmRelayerTransaction;
15use super::{
16 ensure_status, evm_transaction::TX_NONCE_RECONCILE_TRIGGER, get_age_since_status_change,
17 has_enough_confirmations, is_noop, is_too_early_to_resubmit, is_transaction_valid, make_noop,
18 too_many_attempts, too_many_noop_attempts,
19};
20use crate::constants::{
21 get_evm_min_age_for_hash_recovery, get_evm_pending_recovery_trigger_timeout,
22 get_evm_prepare_timeout, get_evm_resend_timeout, ARBITRUM_TIME_TO_RESUBMIT,
23 DEFAULT_EVM_INCLUDE_REVERT_DATA, EVM_MIN_HASHES_FOR_RECOVERY, MAX_GAP_SCAN_RANGE,
24};
25use crate::domain::transaction::common::{
26 get_age_of_sent_at, is_final_state, is_pending_transaction,
27};
28use crate::domain::transaction::util::get_age_since_created;
29use crate::models::{EvmNetwork, NetworkRepoModel, NetworkType};
30use crate::repositories::{NetworkRepository, RelayerRepository};
31use crate::{
32 domain::transaction::evm::price_calculator::PriceCalculatorTrait,
33 jobs::{JobProducerTrait, StatusCheckContext},
34 models::{
35 EvmTransactionData, NetworkTransactionData, RelayerRepoModel, TransactionError,
36 TransactionRepoModel, TransactionStatus, TransactionUpdateRequest,
37 },
38 repositories::{Repository, TransactionCounterTrait, TransactionRepository},
39 services::{provider::EvmProviderTrait, signer::Signer},
40 utils::{get_resubmit_timeout_for_speed, get_resubmit_timeout_with_backoff},
41};
42
43const REVERT_REASON_GENERIC: &str = "Transaction reverted on-chain (receipt status: failed)";
46
47const MAX_REVERT_DATA_HEX_LEN: usize = 4096;
51
52fn truncate_revert_hex(hex: &str) -> String {
55 if hex.len() <= MAX_REVERT_DATA_HEX_LEN {
56 return hex.to_string();
57 }
58 format!("{}...(truncated)", &hex[..MAX_REVERT_DATA_HEX_LEN])
59}
60
61fn build_revert_call_request(
65 evm_data: &EvmTransactionData,
66) -> Result<TransactionRequest, TransactionError> {
67 let from = evm_data.from.parse::<Address>().map_err(|e| {
68 TransactionError::UnexpectedError(format!("Invalid from address for revert recovery: {e}"))
69 })?;
70 let to = match evm_data.to.as_ref() {
71 Some(addr) => TxKind::Call(addr.parse::<Address>().map_err(|e| {
72 TransactionError::UnexpectedError(format!(
73 "Invalid to address for revert recovery: {e}"
74 ))
75 })?),
76 None => TxKind::Create,
77 };
78 let input = evm_data.data_to_bytes().map_err(|e| {
79 TransactionError::UnexpectedError(format!("Invalid input data for revert recovery: {e}"))
80 })?;
81
82 Ok(TransactionRequest {
83 from: Some(from),
84 to: Some(to),
85 value: Some(evm_data.value),
86 input: TransactionInput::from(input),
87 gas: evm_data.gas_limit,
88 gas_price: evm_data.gas_price,
89 max_fee_per_gas: evm_data.max_fee_per_gas,
90 max_priority_fee_per_gas: evm_data.max_priority_fee_per_gas,
91 ..Default::default()
92 })
93}
94
95fn extract_trace_output(trace: &serde_json::Value) -> Option<String> {
98 trace
99 .get("output")
100 .and_then(|v| v.as_str())
101 .filter(|s| s.starts_with("0x") && *s != "0x")
102 .map(|s| s.to_string())
103}
104
105impl<P, RR, NR, TR, J, S, TCR, PC> EvmRelayerTransaction<P, RR, NR, TR, J, S, TCR, PC>
106where
107 P: EvmProviderTrait + Send + Sync,
108 RR: RelayerRepository + Repository<RelayerRepoModel, String> + Send + Sync + 'static,
109 NR: NetworkRepository + Repository<NetworkRepoModel, String> + Send + Sync + 'static,
110 TR: TransactionRepository + Repository<TransactionRepoModel, String> + Send + Sync + 'static,
111 J: JobProducerTrait + Send + Sync + 'static,
112 S: Signer + Send + Sync + 'static,
113 TCR: TransactionCounterTrait + Send + Sync + 'static,
114 PC: PriceCalculatorTrait + Send + Sync,
115{
116 pub(super) async fn check_transaction_status(
117 &self,
118 tx: &TransactionRepoModel,
119 ) -> Result<TransactionStatus, TransactionError> {
120 if is_final_state(&tx.status) {
122 return Ok(tx.status.clone());
123 }
124
125 match tx.status {
128 TransactionStatus::Pending | TransactionStatus::Sent => {
129 return Ok(tx.status.clone());
130 }
131 _ => {}
132 }
133
134 let evm_data = tx.network_data.get_evm_transaction_data()?;
135 let tx_hash = evm_data
136 .hash
137 .as_ref()
138 .ok_or(TransactionError::UnexpectedError(
139 "Transaction hash is missing".to_string(),
140 ))?;
141
142 let receipt_result = self.provider().get_transaction_receipt(tx_hash).await?;
143
144 if let Some(receipt) = receipt_result {
145 if !receipt.inner.status() {
146 return Ok(TransactionStatus::Failed);
147 }
148 let last_block_number = self.provider().get_block_number().await?;
149 let tx_block_number = receipt
150 .block_number
151 .ok_or(TransactionError::UnexpectedError(
152 "Transaction receipt missing block number".to_string(),
153 ))?;
154
155 let network_model = self
156 .network_repository()
157 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
158 .await?
159 .ok_or(TransactionError::UnexpectedError(format!(
160 "Network with chain id {} not found",
161 evm_data.chain_id
162 )))?;
163
164 let network = EvmNetwork::try_from(network_model).map_err(|e| {
165 TransactionError::UnexpectedError(format!(
166 "Error converting network model to EvmNetwork: {e}"
167 ))
168 })?;
169
170 if !has_enough_confirmations(
171 tx_block_number,
172 last_block_number,
173 network.required_confirmations,
174 ) {
175 debug!(
176 tx_id = %tx.id,
177 relayer_id = %tx.relayer_id,
178 tx_hash = %tx_hash,
179 "transaction mined but not confirmed"
180 );
181 return Ok(TransactionStatus::Mined);
182 }
183 Ok(TransactionStatus::Confirmed)
184 } else {
185 debug!(
186 tx_id = %tx.id,
187 relayer_id = %tx.relayer_id,
188 tx_hash = %tx_hash,
189 "transaction not yet mined"
190 );
191
192 if tx.hashes.len() > 1 && self.should_try_hash_recovery(tx)? {
196 if let Some(recovered_tx) = self
197 .try_recover_with_historical_hashes(tx, &evm_data)
198 .await?
199 {
200 return Ok(recovered_tx.status);
202 }
203 }
204
205 Ok(TransactionStatus::Submitted)
206 }
207 }
208
209 pub(super) async fn should_resubmit(
211 &self,
212 tx: &TransactionRepoModel,
213 ) -> Result<bool, TransactionError> {
214 ensure_status(tx, TransactionStatus::Submitted, Some("should_resubmit"))?;
216
217 let evm_data = tx.network_data.get_evm_transaction_data()?;
218 let age = get_age_of_sent_at(tx)?;
219
220 let network_model = self
222 .network_repository()
223 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
224 .await?
225 .ok_or(TransactionError::UnexpectedError(format!(
226 "Network with chain id {} not found",
227 evm_data.chain_id
228 )))?;
229
230 let network = EvmNetwork::try_from(network_model).map_err(|e| {
231 TransactionError::UnexpectedError(format!(
232 "Error converting network model to EvmNetwork: {e}"
233 ))
234 })?;
235
236 let timeout = match network.is_arbitrum() {
237 true => ARBITRUM_TIME_TO_RESUBMIT,
238 false => get_resubmit_timeout_for_speed(&evm_data.speed),
239 };
240
241 let timeout_with_backoff = match network.is_arbitrum() {
242 true => timeout, false => get_resubmit_timeout_with_backoff(timeout, tx.hashes.len()),
244 };
245
246 if age > Duration::milliseconds(timeout_with_backoff) {
247 debug!(
248 tx_id = %tx.id,
249 relayer_id = %tx.relayer_id,
250 age_ms = %age.num_milliseconds(),
251 "transaction has been pending for too long, resubmitting"
252 );
253 return Ok(true);
254 }
255 Ok(false)
256 }
257
258 pub(super) async fn should_noop(
268 &self,
269 tx: &TransactionRepoModel,
270 ) -> Result<(bool, Option<String>), TransactionError> {
271 if too_many_noop_attempts(tx) {
272 debug!("Transaction has too many NOOP attempts already");
273 return Ok((false, None));
274 }
275
276 let evm_data = tx.network_data.get_evm_transaction_data()?;
277 if is_noop(&evm_data) {
278 return Ok((false, None));
279 }
280
281 let network_model = self
282 .network_repository()
283 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
284 .await?
285 .ok_or(TransactionError::UnexpectedError(format!(
286 "Network with chain id {} not found",
287 evm_data.chain_id
288 )))?;
289
290 let network = EvmNetwork::try_from(network_model).map_err(|e| {
291 TransactionError::UnexpectedError(format!(
292 "Error converting network model to EvmNetwork: {e}"
293 ))
294 })?;
295
296 if network.is_rollup() && too_many_attempts(tx) {
297 let reason =
298 "Rollup transaction has too many attempts. Replacing with NOOP.".to_string();
299 debug!(
300 tx_id = %tx.id,
301 relayer_id = %tx.relayer_id,
302 reason = %reason,
303 "replacing transaction with NOOP"
304 );
305 return Ok((true, Some(reason)));
306 }
307
308 if !is_transaction_valid(&tx.created_at, &tx.valid_until) {
309 let reason = "Transaction is expired. Replacing with NOOP.".to_string();
310 debug!(
311 tx_id = %tx.id,
312 relayer_id = %tx.relayer_id,
313 reason = %reason,
314 "replacing transaction with NOOP"
315 );
316 return Ok((true, Some(reason)));
317 }
318
319 if tx.status == TransactionStatus::Pending {
320 let created_at = &tx.created_at;
321 let created_time = DateTime::parse_from_rfc3339(created_at)
322 .map_err(|e| {
323 TransactionError::UnexpectedError(format!("Invalid created_at timestamp: {e}"))
324 })?
325 .with_timezone(&Utc);
326 let age = Utc::now().signed_duration_since(created_time);
327 if age > get_evm_prepare_timeout() {
328 let reason = format!(
329 "Transaction in Pending state for over {} minutes. Replacing with NOOP.",
330 get_evm_prepare_timeout().num_minutes()
331 );
332 debug!(
333 tx_id = %tx.id,
334 relayer_id = %tx.relayer_id,
335 reason = %reason,
336 "replacing transaction with NOOP"
337 );
338 return Ok((true, Some(reason)));
339 }
340 }
341
342 let latest_block = self.provider().get_block_by_number().await;
343 if let Ok(block) = latest_block {
344 let block_gas_limit = block.header.gas_limit;
345 if let Some(gas_limit) = evm_data.gas_limit {
346 if gas_limit > block_gas_limit {
347 let reason = format!(
348 "Transaction gas limit ({gas_limit}) exceeds block gas limit ({block_gas_limit}). Replacing with NOOP.",
349 );
350 warn!(
351 tx_id = %tx.id,
352 tx_gas_limit = %gas_limit,
353 block_gas_limit = %block_gas_limit,
354 "transaction gas limit exceeds block gas limit, replacing with NOOP"
355 );
356 return Ok((true, Some(reason)));
357 }
358 }
359 }
360
361 Ok((false, None))
362 }
363
364 pub(super) async fn update_transaction_status_if_needed(
366 &self,
367 tx: TransactionRepoModel,
368 new_status: TransactionStatus,
369 status_reason: Option<String>,
370 ) -> Result<TransactionRepoModel, TransactionError> {
371 if tx.status != new_status {
372 return self
373 .update_transaction_status(tx, new_status, status_reason)
374 .await;
375 }
376 Ok(tx)
377 }
378
379 pub(super) async fn prepare_noop_update_request(
381 &self,
382 tx: &TransactionRepoModel,
383 is_cancellation: bool,
384 reason: Option<String>,
385 ) -> Result<TransactionUpdateRequest, TransactionError> {
386 let mut evm_data = tx.network_data.get_evm_transaction_data()?;
387 let network_model = self
388 .network_repository()
389 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
390 .await?
391 .ok_or(TransactionError::UnexpectedError(format!(
392 "Network with chain id {} not found",
393 evm_data.chain_id
394 )))?;
395
396 let network = EvmNetwork::try_from(network_model).map_err(|e| {
397 TransactionError::UnexpectedError(format!(
398 "Error converting network model to EvmNetwork: {e}"
399 ))
400 })?;
401
402 make_noop(&mut evm_data, &network, Some(self.provider())).await?;
403
404 let noop_count = tx.noop_count.unwrap_or(0) + 1;
405 let update_request = TransactionUpdateRequest {
406 network_data: Some(NetworkTransactionData::Evm(evm_data)),
407 noop_count: Some(noop_count),
408 status_reason: reason,
409 is_canceled: if is_cancellation {
410 Some(true)
411 } else {
412 tx.is_canceled
413 },
414 ..Default::default()
415 };
416 Ok(update_request)
417 }
418
419 async fn handle_submitted_state(
426 &self,
427 tx: TransactionRepoModel,
428 ) -> Result<TransactionRepoModel, TransactionError> {
429 if self.should_resubmit(&tx).await? {
430 if let Some(nonce_gap_detected) = self.detect_nonce_gap_ahead(&tx).await {
433 if nonce_gap_detected {
434 return self
436 .update_transaction_status_if_needed(tx, TransactionStatus::Submitted, None)
437 .await;
438 }
439 }
440
441 let resubmitted_tx = self.handle_resubmission(tx).await?;
442 return Ok(resubmitted_tx);
443 }
444
445 self.update_transaction_status_if_needed(tx, TransactionStatus::Submitted, None)
446 .await
447 }
448
449 async fn detect_nonce_gap_ahead(&self, tx: &TransactionRepoModel) -> Option<bool> {
463 let evm_data = match tx.network_data.get_evm_transaction_data() {
464 Ok(d) => d,
465 Err(_) => return None,
466 };
467 let tx_nonce = match evm_data.nonce {
468 Some(n) => n,
469 None => return None,
470 };
471
472 let on_chain_nonce = match self
473 .provider()
474 .get_transaction_count(&self.relayer().address)
475 .await
476 {
477 Ok(n) => n,
478 Err(e) => {
479 debug!(
480 tx_id = %tx.id,
481 error = %e,
482 "nonce gap check: failed to get on-chain nonce, skipping"
483 );
484 return None;
485 }
486 };
487
488 if tx_nonce <= on_chain_nonce {
490 return Some(false);
491 }
492
493 let scan_to = std::cmp::min(tx_nonce, on_chain_nonce + MAX_GAP_SCAN_RANGE);
497
498 let occupancy = match self
499 .transaction_repository()
500 .get_nonce_occupancy(&tx.relayer_id, on_chain_nonce, scan_to)
501 .await
502 {
503 Ok(o) => o,
504 Err(e) => {
505 debug!(
506 tx_id = %tx.id,
507 error = %e,
508 "nonce gap check: occupancy lookup failed, skipping"
509 );
510 return None;
511 }
512 };
513
514 let gap_nonces: Vec<u64> = occupancy
515 .into_iter()
516 .filter(|(_, status)| !status.as_ref().is_some_and(is_active_nonce_status))
517 .map(|(nonce, _)| nonce)
518 .collect();
519
520 if gap_nonces.is_empty() {
521 return Some(false);
523 }
524
525 warn!(
526 tx_id = %tx.id,
527 relayer_id = %tx.relayer_id,
528 tx_nonce = tx_nonce,
529 on_chain_nonce = on_chain_nonce,
530 gap_count = gap_nonces.len(),
531 gaps = ?gap_nonces,
532 "nonce gaps confirmed below tx, scheduling nonce health to fill"
533 );
534
535 if let Err(e) = self.schedule_relayer_nonce_health_job(tx).await {
536 warn!(
537 tx_id = %tx.id,
538 error = %e,
539 "failed to schedule nonce health job for nonce gap"
540 );
541 }
542
543 Some(true)
544 }
545
546 async fn handle_resubmission(
548 &self,
549 tx: TransactionRepoModel,
550 ) -> Result<TransactionRepoModel, TransactionError> {
551 debug!(
552 tx_id = %tx.id,
553 relayer_id = %tx.relayer_id,
554 status = ?tx.status,
555 "scheduling resubmit job for transaction"
556 );
557
558 let (should_noop, reason) = self.should_noop(&tx).await?;
560 let tx_to_process = if should_noop {
561 self.process_noop_transaction(&tx, reason).await?
562 } else {
563 tx
564 };
565
566 self.send_transaction_resubmit_job(&tx_to_process).await?;
567 Ok(tx_to_process)
568 }
569
570 async fn process_noop_transaction(
572 &self,
573 tx: &TransactionRepoModel,
574 reason: Option<String>,
575 ) -> Result<TransactionRepoModel, TransactionError> {
576 debug!(
577 tx_id = %tx.id,
578 relayer_id = %tx.relayer_id,
579 status = ?tx.status,
580 "preparing transaction NOOP before resubmission"
581 );
582 let update = self.prepare_noop_update_request(tx, false, reason).await?;
583 let updated_tx = self
584 .transaction_repository()
585 .partial_update(tx.id.clone(), update)
586 .await?;
587
588 let res = self.send_transaction_update_notification(&updated_tx).await;
589 if let Err(e) = res {
590 error!(
591 tx_id = %updated_tx.id,
592 relayer_id = %updated_tx.relayer_id,
593 status = ?updated_tx.status,
594 error = %e,
595 "sending transaction update notification failed for NOOP transaction"
596 );
597 }
598 Ok(updated_tx)
599 }
600
601 async fn handle_pending_state(
603 &self,
604 tx: TransactionRepoModel,
605 ) -> Result<TransactionRepoModel, TransactionError> {
606 let (should_noop, reason) = self.should_noop(&tx).await?;
607 if should_noop {
608 debug!(
611 tx_id = %tx.id,
612 relayer_id = %tx.relayer_id,
613 reason = %reason.as_ref().unwrap_or(&"unknown".to_string()),
614 "marking pending transaction as Failed (nonce not assigned, no NOOP needed)"
615 );
616 let update = TransactionUpdateRequest {
617 status: Some(TransactionStatus::Failed),
618 status_reason: reason,
619 ..Default::default()
620 };
621 let updated_tx = self
622 .transaction_repository()
623 .partial_update(tx.id.clone(), update)
624 .await?;
625
626 let res = self.send_transaction_update_notification(&updated_tx).await;
627 if let Err(e) = res {
628 error!(
629 tx_id = %updated_tx.id,
630 relayer_id = %updated_tx.relayer_id,
631 status = ?updated_tx.status,
632 error = %e,
633 "sending transaction update notification failed for Pending state NOOP"
634 );
635 }
636 return Ok(updated_tx);
637 }
638
639 let age = get_age_since_created(&tx)?;
641 if age > get_evm_pending_recovery_trigger_timeout() {
642 warn!(
643 tx_id = %tx.id,
644 relayer_id = %tx.relayer_id,
645 age_seconds = age.num_seconds(),
646 "transaction stuck in Pending, queuing prepare job"
647 );
648
649 self.send_transaction_request_job(&tx).await?;
651 }
652
653 Ok(tx)
654 }
655
656 async fn handle_mined_state(
658 &self,
659 tx: TransactionRepoModel,
660 ) -> Result<TransactionRepoModel, TransactionError> {
661 self.update_transaction_status_if_needed(tx, TransactionStatus::Mined, None)
662 .await
663 }
664
665 async fn handle_final_state(
667 &self,
668 tx: TransactionRepoModel,
669 status: TransactionStatus,
670 status_reason: Option<String>,
671 ) -> Result<TransactionRepoModel, TransactionError> {
672 self.update_transaction_status_if_needed(tx, status, status_reason)
673 .await
674 }
675
676 async fn mark_as_failed(
678 &self,
679 tx: TransactionRepoModel,
680 reason: String,
681 ) -> Result<TransactionRepoModel, TransactionError> {
682 warn!(
683 tx_id = %tx.id,
684 relayer_id = %tx.relayer_id,
685 reason = %reason,
686 "force-failing transaction due to circuit breaker"
687 );
688
689 let update = TransactionUpdateRequest {
690 status: Some(TransactionStatus::Failed),
691 status_reason: Some(reason),
692 ..Default::default()
693 };
694
695 let updated_tx = self
696 .transaction_repository()
697 .partial_update(tx.id.clone(), update)
698 .await?;
699
700 if let Err(e) = self.send_transaction_update_notification(&updated_tx).await {
702 error!(
703 tx_id = %updated_tx.id,
704 relayer_id = %updated_tx.relayer_id,
705 error = %e,
706 "failed to send notification for force-failed transaction"
707 );
708 }
709
710 Ok(updated_tx)
711 }
712
713 async fn reconcile_tx_nonce_state(
724 &self,
725 tx: &TransactionRepoModel,
726 ) -> Result<Option<TransactionRepoModel>, TransactionError> {
727 let evm_data = tx.network_data.get_evm_transaction_data()?;
728
729 let mut had_rpc_errors = false;
733
734 if let Some(ref hash) = evm_data.hash {
736 match self.provider().get_transaction_receipt(hash).await {
737 Ok(Some(_)) => {
738 debug!(
739 tx_id = %tx.id,
740 hash = %hash,
741 "nonce recovery: receipt found for current hash, deferring to normal flow"
742 );
743 return Ok(None);
744 }
745 Ok(None) => {
746 }
748 Err(e) => {
749 warn!(
750 tx_id = %tx.id,
751 hash = %hash,
752 error = %e,
753 "nonce recovery: error checking receipt for current hash"
754 );
755 had_rpc_errors = true;
756 }
757 }
758 }
759
760 if tx.hashes.len() > 1 {
762 match self.try_recover_with_historical_hashes(tx, &evm_data).await {
763 Ok(Some(recovered_tx)) => {
764 debug!(
765 tx_id = %tx.id,
766 "nonce recovery: recovered transaction via historical hash"
767 );
768 return Ok(Some(recovered_tx));
769 }
770 Ok(None) => {
771 }
773 Err(e) => {
774 warn!(
775 tx_id = %tx.id,
776 error = %e,
777 "nonce recovery: error during historical hash recovery"
778 );
779 had_rpc_errors = true;
780 }
781 }
782 }
783
784 if had_rpc_errors {
788 warn!(
789 tx_id = %tx.id,
790 "nonce recovery: skipping nonce comparison due to RPC errors during hash checks, deferring to normal flow"
791 );
792 return Ok(None);
793 }
794
795 let tx_nonce = match evm_data.nonce {
796 Some(n) => n,
797 None => {
798 return Ok(None);
800 }
801 };
802
803 let on_chain_nonce = self
804 .provider()
805 .get_transaction_count(&self.relayer().address)
806 .await
807 .map_err(|e| {
808 TransactionError::UnexpectedError(format!(
809 "Failed to get on-chain nonce for recovery: {e}"
810 ))
811 })?;
812
813 if on_chain_nonce > tx_nonce {
814 let reason = format!(
816 "Nonce {tx_nonce} consumed externally (on-chain nonce: {on_chain_nonce}). \
817 No matching transaction hash found on-chain."
818 );
819 warn!(
820 tx_id = %tx.id,
821 relayer_id = %tx.relayer_id,
822 tx_nonce = tx_nonce,
823 on_chain_nonce = on_chain_nonce,
824 "nonce recovery: nonce consumed externally, marking as Failed"
825 );
826
827 let updated_tx = self
828 .update_transaction_status(tx.clone(), TransactionStatus::Failed, Some(reason))
829 .await?;
830
831 if let Err(e) = self.schedule_relayer_nonce_health_job(tx).await {
836 warn!(
837 tx_id = %tx.id,
838 error = %e,
839 "nonce recovery: failed to schedule nonce health after external consumption"
840 );
841 }
842
843 return Ok(Some(updated_tx));
844 }
845
846 debug!(
848 tx_id = %tx.id,
849 tx_nonce = tx_nonce,
850 on_chain_nonce = on_chain_nonce,
851 "nonce recovery: on-chain nonce not past tx nonce, deferring to normal flow"
852 );
853 Ok(None)
854 }
855
856 async fn handle_circuit_breaker_safely(
868 &self,
869 tx: TransactionRepoModel,
870 ctx: &StatusCheckContext,
871 ) -> Result<TransactionRepoModel, TransactionError> {
872 let reason = format!(
873 "Transaction status monitoring failed after {} consecutive errors (total: {}). \
874 Last status: {:?}.",
875 ctx.consecutive_failures, ctx.total_failures, tx.status
876 );
877
878 match tx.status {
879 TransactionStatus::Pending => {
880 debug!(
882 tx_id = %tx.id,
883 relayer_id = %tx.relayer_id,
884 "circuit breaker: Pending transaction (no nonce) - safe to mark as Failed"
885 );
886 self.mark_as_failed(tx, reason).await
887 }
888 TransactionStatus::Sent => {
889 let has_nonce = tx
893 .network_data
894 .get_evm_transaction_data()
895 .map(|d| d.nonce.is_some())
896 .unwrap_or(false);
897
898 if has_nonce {
899 warn!(
900 tx_id = %tx.id,
901 relayer_id = %tx.relayer_id,
902 "circuit breaker: Sent transaction with nonce assigned - triggering NOOP to clear nonce slot"
903 );
904 let noop_reason = Some(format!(
905 "{reason}. Replacing with NOOP to clear nonce slot (Sent state with assigned nonce)."
906 ));
907 let updated_tx = self.process_noop_transaction(&tx, noop_reason).await?;
908 self.send_transaction_resubmit_job(&updated_tx).await?;
912 Ok(updated_tx)
913 } else {
914 debug!(
916 tx_id = %tx.id,
917 relayer_id = %tx.relayer_id,
918 "circuit breaker: Sent transaction without nonce - safe to mark as Failed"
919 );
920 self.mark_as_failed(tx, reason).await
921 }
922 }
923 TransactionStatus::Submitted => {
924 warn!(
930 tx_id = %tx.id,
931 relayer_id = %tx.relayer_id,
932 "circuit breaker: Submitted transaction - triggering NOOP to safely clear nonce"
933 );
934 let noop_reason = Some(format!(
935 "{reason}. Replacing with NOOP to clear nonce slot."
936 ));
937 let updated_tx = self.process_noop_transaction(&tx, noop_reason).await?;
938 self.send_transaction_resubmit_job(&updated_tx).await?;
939 Ok(updated_tx)
940 }
941 _ => {
942 debug!(
944 tx_id = %tx.id,
945 relayer_id = %tx.relayer_id,
946 status = ?tx.status,
947 "circuit breaker: unexpected status, returning transaction unchanged"
948 );
949 Ok(tx)
950 }
951 }
952 }
953
954 pub async fn handle_status_impl(
959 &self,
960 tx: TransactionRepoModel,
961 context: Option<StatusCheckContext>,
962 ) -> Result<TransactionRepoModel, TransactionError> {
963 debug!(
964 tx_id = %tx.id,
965 relayer_id = %tx.relayer_id,
966 status = ?tx.status,
967 "checking transaction status"
968 );
969
970 if is_final_state(&tx.status) {
972 debug!(
973 tx_id = %tx.id,
974 relayer_id = %tx.relayer_id,
975 status = ?tx.status,
976 "transaction already in final state"
977 );
978 return Ok(tx);
979 }
980
981 if let Some(ref ctx) = context {
985 let is_noop_tx = tx
986 .network_data
987 .get_evm_transaction_data()
988 .map(|data| is_noop(&data))
989 .unwrap_or(false);
990
991 if ctx.should_force_finalize() && !is_noop_tx {
992 warn!(
993 tx_id = %tx.id,
994 consecutive_failures = ctx.consecutive_failures,
995 total_failures = ctx.total_failures,
996 max_consecutive = ctx.max_consecutive_failures,
997 status = ?tx.status,
998 "circuit breaker triggered - handling safely based on transaction state"
999 );
1000 return self.handle_circuit_breaker_safely(tx, ctx).await;
1001 }
1002
1003 if ctx.should_force_finalize() && is_noop_tx {
1004 debug!(
1005 tx_id = %tx.id,
1006 consecutive_failures = ctx.consecutive_failures,
1007 relayer_id = %tx.relayer_id,
1008 "circuit breaker would trigger but transaction is NOOP - continuing with normal status logic"
1009 );
1010 }
1011 }
1012
1013 if let Some(ref ctx) = context {
1017 if let Some(ref metadata) = ctx.job_metadata {
1018 if let Some(hint) = metadata.get(TX_NONCE_RECONCILE_TRIGGER) {
1019 debug!(
1020 tx_id = %tx.id,
1021 hint = %hint,
1022 "nonce recovery hint detected - performing nonce reconciliation"
1023 );
1024 match self.reconcile_tx_nonce_state(&tx).await {
1025 Ok(Some(recovered_tx)) => {
1026 return Ok(recovered_tx);
1027 }
1028 Ok(None) => {
1029 debug!(
1031 tx_id = %tx.id,
1032 "nonce recovery did not resolve transaction, continuing normal flow"
1033 );
1034 }
1035 Err(e) => {
1036 warn!(
1038 tx_id = %tx.id,
1039 error = %e,
1040 "nonce recovery failed, falling through to normal status flow"
1041 );
1042 }
1043 }
1044 }
1045 }
1046 }
1047
1048 let status = self.check_transaction_status(&tx).await?;
1053
1054 debug!(
1055 tx_id = %tx.id,
1056 previous_status = ?tx.status,
1057 new_status = ?status,
1058 relayer_id = %tx.relayer_id,
1059 "transaction status check completed"
1060 );
1061
1062 let tx = if status != tx.status {
1066 debug!(
1067 tx_id = %tx.id,
1068 old_status = ?tx.status,
1069 new_status = ?status,
1070 relayer_id = %tx.relayer_id,
1071 "status changed during check, reloading transaction from DB to ensure fresh data"
1072 );
1073 self.transaction_repository()
1074 .get_by_id(tx.id.clone())
1075 .await?
1076 } else {
1077 tx
1078 };
1079
1080 if is_too_early_to_resubmit(&tx)? && is_pending_transaction(&status) {
1085 return self
1087 .update_transaction_status_if_needed(tx, status, None)
1088 .await;
1089 }
1090
1091 match status {
1093 TransactionStatus::Pending => self.handle_pending_state(tx).await,
1094 TransactionStatus::Sent => self.handle_sent_state(tx).await,
1095 TransactionStatus::Submitted => self.handle_submitted_state(tx).await,
1096 TransactionStatus::Mined => self.handle_mined_state(tx).await,
1097 TransactionStatus::Failed => {
1098 let status_reason = if tx.status != TransactionStatus::Failed {
1101 Some(self.build_failed_status_reason(&tx).await)
1102 } else {
1103 None
1104 };
1105 self.handle_final_state(tx, status, status_reason).await
1106 }
1107 TransactionStatus::Confirmed
1108 | TransactionStatus::Expired
1109 | TransactionStatus::Canceled => self.handle_final_state(tx, status, None).await,
1110 }
1111 }
1112
1113 async fn build_failed_status_reason(&self, tx: &TransactionRepoModel) -> String {
1117 if !self
1118 .relayer()
1119 .policies
1120 .get_evm_policy()
1121 .include_revert_data
1122 .unwrap_or(DEFAULT_EVM_INCLUDE_REVERT_DATA)
1123 {
1124 debug!(
1125 tx_id = %tx.id,
1126 "revert-data recovery disabled by policy; using generic reason"
1127 );
1128 return REVERT_REASON_GENERIC.to_string();
1129 }
1130
1131 match self.recover_revert_data(tx).await {
1132 Some(hex) => {
1133 let hex = truncate_revert_hex(&hex);
1134 format!("Transaction reverted on-chain (revert_data: {hex})")
1135 }
1136 None => REVERT_REASON_GENERIC.to_string(),
1137 }
1138 }
1139
1140 async fn recover_revert_data(&self, tx: &TransactionRepoModel) -> Option<String> {
1147 let evm_data = tx.network_data.get_evm_transaction_data().ok()?;
1148 let hash = evm_data.hash.as_ref()?;
1149
1150 match self
1153 .provider()
1154 .raw_request_dyn(
1155 "debug_traceTransaction",
1156 json!([hash, {"tracer": "callTracer"}]),
1157 )
1158 .await
1159 {
1160 Ok(trace) => {
1161 if let Some(hex) = extract_trace_output(&trace) {
1162 info!(
1163 tx_id = %tx.id,
1164 method = "trace",
1165 "recovered on-chain revert data"
1166 );
1167 return Some(hex);
1168 }
1169 debug!(
1170 tx_id = %tx.id,
1171 "trace returned no revert output; trying eth_call fallback"
1172 );
1173 }
1174 Err(e) => {
1175 warn!(
1176 tx_id = %tx.id,
1177 error = %e,
1178 "debug_traceTransaction unavailable; trying eth_call fallback"
1179 );
1180 }
1181 }
1182
1183 let block_number = match self.provider().get_transaction_receipt(hash).await {
1186 Ok(Some(receipt)) => match receipt.block_number {
1187 Some(bn) => bn,
1188 None => {
1189 debug!(
1190 tx_id = %tx.id,
1191 "receipt missing block number; skipping eth_call recovery"
1192 );
1193 return None;
1194 }
1195 },
1196 Ok(None) => {
1197 debug!(
1198 tx_id = %tx.id,
1199 "no receipt found; skipping eth_call recovery"
1200 );
1201 return None;
1202 }
1203 Err(e) => {
1204 warn!(
1205 tx_id = %tx.id,
1206 error = %e,
1207 "failed to re-fetch receipt for revert recovery"
1208 );
1209 return None;
1210 }
1211 };
1212
1213 let request = match build_revert_call_request(&evm_data) {
1214 Ok(req) => req,
1215 Err(e) => {
1216 warn!(
1217 tx_id = %tx.id,
1218 error = %e,
1219 "failed to reconstruct eth_call request for revert recovery"
1220 );
1221 return None;
1222 }
1223 };
1224
1225 match self
1226 .provider()
1227 .get_call_revert_data(&request, block_number)
1228 .await
1229 {
1230 Ok(Some(bytes)) if !bytes.is_empty() => {
1231 let hex = format!("0x{}", hex::encode(&bytes));
1232 info!(
1233 tx_id = %tx.id,
1234 method = "eth_call",
1235 "recovered on-chain revert data"
1236 );
1237 Some(hex)
1238 }
1239 Ok(_) => {
1240 debug!(tx_id = %tx.id, "no revert data recovered");
1241 None
1242 }
1243 Err(e) => {
1244 warn!(
1245 tx_id = %tx.id,
1246 error = %e,
1247 "eth_call revert recovery failed"
1248 );
1249 None
1250 }
1251 }
1252 }
1253
1254 async fn handle_sent_state(
1256 &self,
1257 tx: TransactionRepoModel,
1258 ) -> Result<TransactionRepoModel, TransactionError> {
1259 debug!(
1260 tx_id = %tx.id,
1261 relayer_id = %tx.relayer_id,
1262 "handling Sent state"
1263 );
1264
1265 let (should_noop, reason) = self.should_noop(&tx).await?;
1267 if should_noop {
1268 debug!(
1269 tx_id = %tx.id,
1270 relayer_id = %tx.relayer_id,
1271 "preparing NOOP for sent transaction"
1272 );
1273 let update = self.prepare_noop_update_request(&tx, false, reason).await?;
1274 let updated_tx = self
1275 .transaction_repository()
1276 .partial_update(tx.id.clone(), update)
1277 .await?;
1278
1279 self.send_transaction_submit_job(&updated_tx).await?;
1280 let res = self.send_transaction_update_notification(&updated_tx).await;
1281 if let Err(e) = res {
1282 error!(
1283 tx_id = %updated_tx.id,
1284 relayer_id = %updated_tx.relayer_id,
1285 status = ?updated_tx.status,
1286 error = %e,
1287 "sending transaction update notification failed for Sent state NOOP"
1288 );
1289 }
1290 return Ok(updated_tx);
1291 }
1292
1293 let age_since_sent = get_age_since_status_change(&tx)?;
1296
1297 if age_since_sent > get_evm_resend_timeout() {
1298 warn!(
1299 tx_id = %tx.id,
1300 relayer_id = %tx.relayer_id,
1301 age_seconds = age_since_sent.num_seconds(),
1302 "transaction stuck in Sent, queuing resubmit job with repricing"
1303 );
1304
1305 self.send_transaction_resubmit_job(&tx).await?;
1307 }
1308
1309 self.update_transaction_status_if_needed(tx, TransactionStatus::Sent, None)
1310 .await
1311 }
1312
1313 fn should_try_hash_recovery(
1320 &self,
1321 tx: &TransactionRepoModel,
1322 ) -> Result<bool, TransactionError> {
1323 if tx.status != TransactionStatus::Submitted {
1325 return Ok(false);
1326 }
1327
1328 if tx.hashes.len() <= 1 {
1330 return Ok(false);
1331 }
1332
1333 let age = get_age_of_sent_at(tx)?;
1335 let min_age_for_recovery = get_evm_min_age_for_hash_recovery();
1336
1337 if age < min_age_for_recovery {
1338 return Ok(false);
1339 }
1340
1341 if tx.hashes.len() < EVM_MIN_HASHES_FOR_RECOVERY {
1344 return Ok(false);
1345 }
1346
1347 Ok(true)
1348 }
1349
1350 async fn try_recover_with_historical_hashes(
1359 &self,
1360 tx: &TransactionRepoModel,
1361 evm_data: &crate::models::EvmTransactionData,
1362 ) -> Result<Option<TransactionRepoModel>, TransactionError> {
1363 warn!(
1364 tx_id = %tx.id,
1365 relayer_id = %tx.relayer_id,
1366 current_hash = ?evm_data.hash,
1367 total_hashes = %tx.hashes.len(),
1368 "attempting hash recovery - checking historical hashes"
1369 );
1370
1371 for (idx, historical_hash) in tx.hashes.iter().rev().enumerate() {
1373 if Some(historical_hash) == evm_data.hash.as_ref() {
1375 continue;
1376 }
1377
1378 debug!(
1379 tx_id = %tx.id,
1380 relayer_id = %tx.relayer_id,
1381 hash = %historical_hash,
1382 index = %idx,
1383 "checking historical hash"
1384 );
1385
1386 match self
1388 .provider()
1389 .get_transaction_receipt(historical_hash)
1390 .await
1391 {
1392 Ok(Some(receipt)) => {
1393 warn!(
1394 tx_id = %tx.id,
1395 relayer_id = %tx.relayer_id,
1396 mined_hash = %historical_hash,
1397 wrong_hash = ?evm_data.hash,
1398 block_number = ?receipt.block_number,
1399 "RECOVERED: found mined transaction with historical hash - correcting database"
1400 );
1401
1402 let updated_tx = self
1405 .update_transaction_with_corrected_hash(
1406 tx,
1407 evm_data,
1408 historical_hash,
1409 TransactionStatus::Mined,
1410 )
1411 .await?;
1412
1413 return Ok(Some(updated_tx));
1414 }
1415 Ok(None) => {
1416 continue;
1418 }
1419 Err(e) => {
1420 warn!(
1422 tx_id = %tx.id,
1423 relayer_id = %tx.relayer_id,
1424 hash = %historical_hash,
1425 error = %e,
1426 "error checking historical hash, continuing to next"
1427 );
1428 continue;
1429 }
1430 }
1431 }
1432
1433 debug!(
1435 tx_id = %tx.id,
1436 relayer_id = %tx.relayer_id,
1437 "hash recovery completed - no historical hashes found on-chain"
1438 );
1439 Ok(None)
1440 }
1441
1442 async fn update_transaction_with_corrected_hash(
1446 &self,
1447 tx: &TransactionRepoModel,
1448 evm_data: &crate::models::EvmTransactionData,
1449 correct_hash: &str,
1450 status: TransactionStatus,
1451 ) -> Result<TransactionRepoModel, TransactionError> {
1452 let mut corrected_data = evm_data.clone();
1453 corrected_data.hash = Some(correct_hash.to_string());
1454
1455 let updated_tx = self
1456 .transaction_repository()
1457 .partial_update(
1458 tx.id.clone(),
1459 TransactionUpdateRequest {
1460 network_data: Some(NetworkTransactionData::Evm(corrected_data)),
1461 status: Some(status),
1462 ..Default::default()
1463 },
1464 )
1465 .await?;
1466
1467 if let Err(e) = self.send_transaction_update_notification(&updated_tx).await {
1469 error!(
1470 tx_id = %updated_tx.id,
1471 relayer_id = %updated_tx.relayer_id,
1472 error = %e,
1473 "failed to send notification for hash recovery"
1474 );
1475 }
1476
1477 Ok(updated_tx)
1478 }
1479}
1480
1481#[cfg(test)]
1482mod tests {
1483 use crate::{
1484 config::{EvmNetworkConfig, NetworkConfigCommon},
1485 domain::transaction::evm::{EvmRelayerTransaction, MockPriceCalculatorTrait},
1486 jobs::MockJobProducerTrait,
1487 models::{
1488 evm::Speed, EvmTransactionData, NetworkConfigData, NetworkRepoModel,
1489 NetworkTransactionData, NetworkType, RelayerEvmPolicy, RelayerNetworkPolicy,
1490 RelayerRepoModel, RpcConfig, TransactionReceipt, TransactionRepoModel,
1491 TransactionStatus, U256,
1492 },
1493 repositories::{
1494 MockNetworkRepository, MockRelayerRepository, MockTransactionCounterTrait,
1495 MockTransactionRepository,
1496 },
1497 services::{provider::MockEvmProviderTrait, signer::MockSigner},
1498 };
1499 use alloy::{
1500 consensus::{Eip658Value, Receipt, ReceiptWithBloom},
1501 network::AnyReceiptEnvelope,
1502 primitives::{b256, Address, BlockHash, Bloom, TxHash},
1503 };
1504 use chrono::{Duration, Utc};
1505 use std::sync::Arc;
1506
1507 pub struct TestMocks {
1509 pub provider: MockEvmProviderTrait,
1510 pub relayer_repo: MockRelayerRepository,
1511 pub network_repo: MockNetworkRepository,
1512 pub tx_repo: MockTransactionRepository,
1513 pub job_producer: MockJobProducerTrait,
1514 pub signer: MockSigner,
1515 pub counter: MockTransactionCounterTrait,
1516 pub price_calc: MockPriceCalculatorTrait,
1517 }
1518
1519 pub fn default_test_mocks() -> TestMocks {
1522 TestMocks {
1523 provider: MockEvmProviderTrait::new(),
1524 relayer_repo: MockRelayerRepository::new(),
1525 network_repo: MockNetworkRepository::new(),
1526 tx_repo: MockTransactionRepository::new(),
1527 job_producer: MockJobProducerTrait::new(),
1528 signer: MockSigner::new(),
1529 counter: MockTransactionCounterTrait::new(),
1530 price_calc: MockPriceCalculatorTrait::new(),
1531 }
1532 }
1533
1534 pub fn default_test_mocks_with_network() -> TestMocks {
1536 let mut mocks = default_test_mocks();
1537 mocks
1539 .network_repo
1540 .expect_get_by_chain_id()
1541 .returning(|network_type, chain_id| {
1542 if network_type == NetworkType::Evm && chain_id == 1 {
1543 Ok(Some(create_test_network_model()))
1544 } else {
1545 Ok(None)
1546 }
1547 });
1548 mocks
1549 }
1550
1551 pub fn create_test_network_model() -> NetworkRepoModel {
1553 let evm_config = EvmNetworkConfig {
1554 common: NetworkConfigCommon {
1555 network: "mainnet".to_string(),
1556 from: None,
1557 rpc_urls: Some(vec![RpcConfig::new("https://rpc.example.com".to_string())]),
1558 explorer_urls: Some(vec!["https://explorer.example.com".to_string()]),
1559 average_blocktime_ms: Some(12000),
1560 is_testnet: Some(false),
1561 tags: Some(vec!["mainnet".to_string()]),
1562 },
1563 chain_id: Some(1),
1564 required_confirmations: Some(12),
1565 features: Some(vec!["eip1559".to_string()]),
1566 symbol: Some("ETH".to_string()),
1567 gas_price_cache: None,
1568 };
1569 NetworkRepoModel {
1570 id: "evm:mainnet".to_string(),
1571 name: "mainnet".to_string(),
1572 network_type: NetworkType::Evm,
1573 config: NetworkConfigData::Evm(evm_config),
1574 }
1575 }
1576
1577 pub fn create_test_no_mempool_network_model() -> NetworkRepoModel {
1579 let evm_config = EvmNetworkConfig {
1580 common: NetworkConfigCommon {
1581 network: "arbitrum".to_string(),
1582 from: None,
1583 rpc_urls: Some(vec![crate::models::RpcConfig::new(
1584 "https://arb-rpc.example.com".to_string(),
1585 )]),
1586 explorer_urls: Some(vec!["https://arb-explorer.example.com".to_string()]),
1587 average_blocktime_ms: Some(1000),
1588 is_testnet: Some(false),
1589 tags: Some(vec![
1590 "arbitrum".to_string(),
1591 "rollup".to_string(),
1592 "no-mempool".to_string(),
1593 ]),
1594 },
1595 chain_id: Some(42161),
1596 required_confirmations: Some(12),
1597 features: Some(vec!["eip1559".to_string()]),
1598 symbol: Some("ETH".to_string()),
1599 gas_price_cache: None,
1600 };
1601 NetworkRepoModel {
1602 id: "evm:arbitrum".to_string(),
1603 name: "arbitrum".to_string(),
1604 network_type: NetworkType::Evm,
1605 config: NetworkConfigData::Evm(evm_config),
1606 }
1607 }
1608
1609 pub fn make_test_transaction(status: TransactionStatus) -> TransactionRepoModel {
1613 TransactionRepoModel {
1614 id: "test-tx-id".to_string(),
1615 relayer_id: "test-relayer-id".to_string(),
1616 status,
1617 status_reason: None,
1618 created_at: Utc::now().to_rfc3339(),
1619 sent_at: None,
1620 confirmed_at: None,
1621 valid_until: None,
1622 delete_at: None,
1623 network_type: NetworkType::Evm,
1624 network_data: NetworkTransactionData::Evm(EvmTransactionData {
1625 chain_id: 1,
1626 from: "0xSender".to_string(),
1627 to: Some("0xRecipient".to_string()),
1628 value: U256::from(0),
1629 data: Some("0xData".to_string()),
1630 gas_limit: Some(21000),
1631 gas_price: Some(20000000000),
1632 max_fee_per_gas: None,
1633 max_priority_fee_per_gas: None,
1634 nonce: None,
1635 signature: None,
1636 hash: None,
1637 speed: Some(Speed::Fast),
1638 raw: None,
1639 }),
1640 priced_at: None,
1641 hashes: Vec::new(),
1642 noop_count: None,
1643 is_canceled: Some(false),
1644 metadata: None,
1645 }
1646 }
1647
1648 pub fn make_test_evm_relayer_transaction(
1651 relayer: RelayerRepoModel,
1652 mocks: TestMocks,
1653 ) -> EvmRelayerTransaction<
1654 MockEvmProviderTrait,
1655 MockRelayerRepository,
1656 MockNetworkRepository,
1657 MockTransactionRepository,
1658 MockJobProducerTrait,
1659 MockSigner,
1660 MockTransactionCounterTrait,
1661 MockPriceCalculatorTrait,
1662 > {
1663 EvmRelayerTransaction::new(
1664 relayer,
1665 mocks.provider,
1666 Arc::new(mocks.relayer_repo),
1667 Arc::new(mocks.network_repo),
1668 Arc::new(mocks.tx_repo),
1669 Arc::new(mocks.counter),
1670 Arc::new(mocks.job_producer),
1671 mocks.price_calc,
1672 mocks.signer,
1673 )
1674 .unwrap()
1675 }
1676
1677 fn create_test_relayer() -> RelayerRepoModel {
1678 RelayerRepoModel {
1679 id: "test-relayer-id".to_string(),
1680 name: "Test Relayer".to_string(),
1681 paused: false,
1682 system_disabled: false,
1683 network: "test_network".to_string(),
1684 network_type: NetworkType::Evm,
1685 policies: RelayerNetworkPolicy::Evm(RelayerEvmPolicy::default()),
1686 signer_id: "test_signer".to_string(),
1687 address: "0x".to_string(),
1688 notification_id: None,
1689 custom_rpc_urls: None,
1690 ..Default::default()
1691 }
1692 }
1693
1694 fn make_mock_receipt(status: bool, block_number: Option<u64>) -> TransactionReceipt {
1695 let tx_hash = TxHash::from(b256!(
1697 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
1698 ));
1699 let block_hash = BlockHash::from(b256!(
1700 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
1701 ));
1702 let from_address = Address::from([0x11; 20]);
1703
1704 TransactionReceipt {
1705 inner: alloy::rpc::types::TransactionReceipt {
1706 inner: AnyReceiptEnvelope {
1707 inner: ReceiptWithBloom {
1708 receipt: Receipt {
1709 status: Eip658Value::Eip658(status), cumulative_gas_used: 0,
1711 logs: vec![],
1712 },
1713 logs_bloom: Bloom::ZERO,
1714 },
1715 r#type: 0, },
1717 transaction_hash: tx_hash,
1718 transaction_index: Some(0),
1719 block_hash: block_number.map(|_| block_hash), block_number,
1721 gas_used: 21000,
1722 effective_gas_price: 1000,
1723 blob_gas_used: None,
1724 blob_gas_price: None,
1725 from: from_address,
1726 to: None,
1727 contract_address: None,
1728 },
1729 other: Default::default(),
1730 }
1731 }
1732
1733 mod check_transaction_status_tests {
1735 use super::*;
1736
1737 #[tokio::test]
1738 async fn test_not_mined() {
1739 let mut mocks = default_test_mocks();
1740 let relayer = create_test_relayer();
1741 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1742
1743 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1745 evm_data.hash = Some("0xFakeHash".to_string());
1746 }
1747
1748 mocks
1750 .provider
1751 .expect_get_transaction_receipt()
1752 .returning(|_| Box::pin(async { Ok(None) }));
1753
1754 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1755
1756 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1757 assert_eq!(status, TransactionStatus::Submitted);
1758 }
1759
1760 #[tokio::test]
1761 async fn test_mined_but_not_confirmed() {
1762 let mut mocks = default_test_mocks();
1763 let relayer = create_test_relayer();
1764 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1765
1766 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1767 evm_data.hash = Some("0xFakeHash".to_string());
1768 }
1769
1770 mocks
1772 .provider
1773 .expect_get_transaction_receipt()
1774 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1775
1776 mocks
1778 .provider
1779 .expect_get_block_number()
1780 .return_once(|| Box::pin(async { Ok(100) }));
1781
1782 mocks
1784 .network_repo
1785 .expect_get_by_chain_id()
1786 .returning(|_, _| Ok(Some(create_test_network_model())));
1787
1788 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1789
1790 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1791 assert_eq!(status, TransactionStatus::Mined);
1792 }
1793
1794 #[tokio::test]
1795 async fn test_confirmed() {
1796 let mut mocks = default_test_mocks();
1797 let relayer = create_test_relayer();
1798 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1799
1800 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1801 evm_data.hash = Some("0xFakeHash".to_string());
1802 }
1803
1804 mocks
1806 .provider
1807 .expect_get_transaction_receipt()
1808 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1809
1810 mocks
1812 .provider
1813 .expect_get_block_number()
1814 .return_once(|| Box::pin(async { Ok(113) }));
1815
1816 mocks
1818 .network_repo
1819 .expect_get_by_chain_id()
1820 .returning(|_, _| Ok(Some(create_test_network_model())));
1821
1822 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1823
1824 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1825 assert_eq!(status, TransactionStatus::Confirmed);
1826 }
1827
1828 #[tokio::test]
1829 async fn test_failed() {
1830 let mut mocks = default_test_mocks();
1831 let relayer = create_test_relayer();
1832 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1833
1834 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1835 evm_data.hash = Some("0xFakeHash".to_string());
1836 }
1837
1838 mocks
1840 .provider
1841 .expect_get_transaction_receipt()
1842 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
1843
1844 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1845
1846 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1847 assert_eq!(status, TransactionStatus::Failed);
1848 }
1849 }
1850
1851 mod should_resubmit_tests {
1853 use super::*;
1854 use crate::models::TransactionError;
1855
1856 #[tokio::test]
1857 async fn test_should_resubmit_true() {
1858 let mut mocks = default_test_mocks();
1859 let relayer = create_test_relayer();
1860
1861 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1863 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1864
1865 mocks
1867 .network_repo
1868 .expect_get_by_chain_id()
1869 .returning(|_, _| Ok(Some(create_test_network_model())));
1870
1871 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1872 let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1873 assert!(res, "Transaction should be resubmitted after timeout.");
1874 }
1875
1876 #[tokio::test]
1877 async fn test_should_resubmit_false() {
1878 let mut mocks = default_test_mocks();
1879 let relayer = create_test_relayer();
1880
1881 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1883 tx.sent_at = Some(Utc::now().to_rfc3339());
1884
1885 mocks
1887 .network_repo
1888 .expect_get_by_chain_id()
1889 .returning(|_, _| Ok(Some(create_test_network_model())));
1890
1891 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1892 let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1893 assert!(!res, "Transaction should not be resubmitted immediately.");
1894 }
1895
1896 #[tokio::test]
1897 async fn test_should_resubmit_true_for_no_mempool_network() {
1898 let mut mocks = default_test_mocks();
1899 let relayer = create_test_relayer();
1900
1901 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1903 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1904
1905 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1907 evm_data.chain_id = 42161; }
1909
1910 mocks
1912 .network_repo
1913 .expect_get_by_chain_id()
1914 .returning(|_, _| Ok(Some(create_test_no_mempool_network_model())));
1915
1916 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1917 let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1918 assert!(
1919 res,
1920 "Transaction should be resubmitted for no-mempool networks."
1921 );
1922 }
1923
1924 #[tokio::test]
1925 async fn test_should_resubmit_network_not_found() {
1926 let mut mocks = default_test_mocks();
1927 let relayer = create_test_relayer();
1928
1929 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1930 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1931
1932 mocks
1934 .network_repo
1935 .expect_get_by_chain_id()
1936 .returning(|_, _| Ok(None));
1937
1938 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1939 let result = evm_transaction.should_resubmit(&tx).await;
1940
1941 assert!(
1942 result.is_err(),
1943 "should_resubmit should return error when network not found"
1944 );
1945 let error = result.unwrap_err();
1946 match error {
1947 TransactionError::UnexpectedError(msg) => {
1948 assert!(msg.contains("Network with chain id 1 not found"));
1949 }
1950 _ => panic!("Expected UnexpectedError for network not found"),
1951 }
1952 }
1953
1954 #[tokio::test]
1955 async fn test_should_resubmit_network_conversion_error() {
1956 let mut mocks = default_test_mocks();
1957 let relayer = create_test_relayer();
1958
1959 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1960 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1961
1962 let invalid_evm_config = EvmNetworkConfig {
1964 common: NetworkConfigCommon {
1965 network: "invalid-network".to_string(),
1966 from: None,
1967 rpc_urls: Some(vec![crate::models::RpcConfig::new(
1968 "https://rpc.example.com".to_string(),
1969 )]),
1970 explorer_urls: Some(vec!["https://explorer.example.com".to_string()]),
1971 average_blocktime_ms: Some(12000),
1972 is_testnet: Some(false),
1973 tags: Some(vec!["testnet".to_string()]),
1974 },
1975 chain_id: None, required_confirmations: Some(12),
1977 features: Some(vec!["eip1559".to_string()]),
1978 symbol: Some("ETH".to_string()),
1979 gas_price_cache: None,
1980 };
1981 let invalid_network = NetworkRepoModel {
1982 id: "evm:invalid".to_string(),
1983 name: "invalid-network".to_string(),
1984 network_type: NetworkType::Evm,
1985 config: NetworkConfigData::Evm(invalid_evm_config),
1986 };
1987
1988 mocks
1990 .network_repo
1991 .expect_get_by_chain_id()
1992 .returning(move |_, _| Ok(Some(invalid_network.clone())));
1993
1994 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1995 let result = evm_transaction.should_resubmit(&tx).await;
1996
1997 assert!(
1998 result.is_err(),
1999 "should_resubmit should return error when network conversion fails"
2000 );
2001 let error = result.unwrap_err();
2002 match error {
2003 TransactionError::UnexpectedError(msg) => {
2004 assert!(msg.contains("Error converting network model to EvmNetwork"));
2005 }
2006 _ => panic!("Expected UnexpectedError for network conversion failure"),
2007 }
2008 }
2009 }
2010
2011 mod should_noop_tests {
2013 use super::*;
2014
2015 #[tokio::test]
2016 async fn test_expired_transaction_triggers_noop() {
2017 let mut mocks = default_test_mocks();
2018 let relayer = create_test_relayer();
2019
2020 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2021 tx.valid_until = Some((Utc::now() - Duration::seconds(10)).to_rfc3339());
2023
2024 mocks
2026 .network_repo
2027 .expect_get_by_chain_id()
2028 .returning(|_, _| Ok(Some(create_test_network_model())));
2029
2030 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2031 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2032 assert!(res, "Expired transaction should be replaced with a NOOP.");
2033 assert!(
2034 reason.is_some(),
2035 "Reason should be provided for expired transaction"
2036 );
2037 assert!(
2038 reason.unwrap().contains("expired"),
2039 "Reason should mention expiration"
2040 );
2041 }
2042
2043 #[tokio::test]
2044 async fn test_too_many_noop_attempts_returns_false() {
2045 let mocks = default_test_mocks();
2046 let relayer = create_test_relayer();
2047
2048 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2049 tx.noop_count = Some(51); let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2052 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2053 assert!(
2054 !res,
2055 "Transaction with too many NOOP attempts should not be replaced."
2056 );
2057 assert!(
2058 reason.is_none(),
2059 "Reason should not be provided when should_noop is false"
2060 );
2061 }
2062
2063 #[tokio::test]
2064 async fn test_already_noop_returns_false() {
2065 let mut mocks = default_test_mocks();
2066 let relayer = create_test_relayer();
2067
2068 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2069 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2071 evm_data.to = None;
2072 evm_data.value = U256::from(0);
2073 }
2074
2075 mocks
2076 .network_repo
2077 .expect_get_by_chain_id()
2078 .returning(|_, _| Ok(Some(create_test_network_model())));
2079
2080 mocks.provider.expect_get_block_by_number().returning(|| {
2082 Box::pin(async {
2083 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2084 let mut block: Block = Block::default();
2085 block.header.gas_limit = 30_000_000u64;
2086 Ok(AnyRpcBlock::from(block))
2087 })
2088 });
2089
2090 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2091 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2092 assert!(
2093 !res,
2094 "Transaction that is already a NOOP should not be replaced."
2095 );
2096 assert!(
2097 reason.is_none(),
2098 "Reason should not be provided when should_noop is false"
2099 );
2100 }
2101
2102 #[tokio::test]
2103 async fn test_rollup_with_too_many_attempts_triggers_noop() {
2104 let mut mocks = default_test_mocks();
2105 let relayer = create_test_relayer();
2106
2107 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2108 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2110 evm_data.chain_id = 42161; }
2112 tx.hashes = vec!["0xHash1".to_string(); 51];
2114
2115 mocks
2117 .network_repo
2118 .expect_get_by_chain_id()
2119 .returning(|_, _| Ok(Some(create_test_no_mempool_network_model())));
2120
2121 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2122 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2123 assert!(
2124 res,
2125 "Rollup transaction with too many attempts should be replaced with NOOP."
2126 );
2127 assert!(
2128 reason.is_some(),
2129 "Reason should be provided for rollup transaction"
2130 );
2131 assert!(
2132 reason.unwrap().contains("too many attempts"),
2133 "Reason should mention too many attempts"
2134 );
2135 }
2136
2137 #[tokio::test]
2138 async fn test_pending_state_timeout_triggers_noop() {
2139 let mut mocks = default_test_mocks();
2140 let relayer = create_test_relayer();
2141
2142 let mut tx = make_test_transaction(TransactionStatus::Pending);
2143 tx.created_at = (Utc::now() - Duration::minutes(3)).to_rfc3339();
2145
2146 mocks
2147 .network_repo
2148 .expect_get_by_chain_id()
2149 .returning(|_, _| Ok(Some(create_test_network_model())));
2150
2151 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2152 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2153 assert!(
2154 res,
2155 "Pending transaction stuck for >2 minutes should be replaced with NOOP."
2156 );
2157 assert!(
2158 reason.is_some(),
2159 "Reason should be provided for pending timeout"
2160 );
2161 assert!(
2162 reason.unwrap().contains("Pending state"),
2163 "Reason should mention Pending state"
2164 );
2165 }
2166
2167 #[tokio::test]
2168 async fn test_valid_transaction_returns_false() {
2169 let mut mocks = default_test_mocks();
2170 let relayer = create_test_relayer();
2171
2172 let tx = make_test_transaction(TransactionStatus::Submitted);
2173 mocks
2176 .network_repo
2177 .expect_get_by_chain_id()
2178 .returning(|_, _| Ok(Some(create_test_network_model())));
2179
2180 mocks.provider.expect_get_block_by_number().returning(|| {
2182 Box::pin(async {
2183 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2184 let mut block: Block = Block::default();
2185 block.header.gas_limit = 30_000_000u64;
2186 Ok(AnyRpcBlock::from(block))
2187 })
2188 });
2189
2190 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2191 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2192 assert!(!res, "Valid transaction should not be replaced with NOOP.");
2193 assert!(
2194 reason.is_none(),
2195 "Reason should not be provided when should_noop is false"
2196 );
2197 }
2198 }
2199
2200 mod update_transaction_status_tests {
2202 use super::*;
2203
2204 #[tokio::test]
2205 async fn test_no_update_when_status_is_same() {
2206 let mocks = default_test_mocks();
2208 let relayer = create_test_relayer();
2209 let tx = make_test_transaction(TransactionStatus::Submitted);
2210 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2211
2212 let updated_tx = evm_transaction
2215 .update_transaction_status_if_needed(tx.clone(), TransactionStatus::Submitted, None)
2216 .await
2217 .unwrap();
2218 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2219 assert_eq!(updated_tx.id, tx.id);
2220 }
2221
2222 #[tokio::test]
2223 async fn test_updates_when_status_differs() {
2224 let mut mocks = default_test_mocks();
2225 let relayer = create_test_relayer();
2226 let tx = make_test_transaction(TransactionStatus::Submitted);
2227
2228 mocks
2230 .tx_repo
2231 .expect_partial_update()
2232 .returning(|_, update| {
2233 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2234 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2235 Ok(updated_tx)
2236 });
2237
2238 mocks
2240 .job_producer
2241 .expect_produce_send_notification_job()
2242 .returning(|_, _| Box::pin(async { Ok(()) }));
2243
2244 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2245 let updated_tx = evm_transaction
2246 .update_transaction_status_if_needed(tx.clone(), TransactionStatus::Mined, None)
2247 .await
2248 .unwrap();
2249
2250 assert_eq!(updated_tx.status, TransactionStatus::Mined);
2251 }
2252
2253 #[tokio::test]
2254 async fn test_updates_with_status_reason() {
2255 let mut mocks = default_test_mocks();
2256 let relayer = create_test_relayer();
2257 let tx = make_test_transaction(TransactionStatus::Submitted);
2258
2259 mocks
2260 .tx_repo
2261 .expect_partial_update()
2262 .withf(|_, update| {
2263 update.status == Some(TransactionStatus::Failed)
2264 && update.status_reason == Some("Transaction reverted on-chain".to_string())
2265 })
2266 .returning(|_, update| {
2267 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2268 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2269 updated_tx.status_reason = update.status_reason.clone();
2270 Ok(updated_tx)
2271 });
2272
2273 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2274 let updated_tx = evm_transaction
2275 .update_transaction_status_if_needed(
2276 tx.clone(),
2277 TransactionStatus::Failed,
2278 Some("Transaction reverted on-chain".to_string()),
2279 )
2280 .await
2281 .unwrap();
2282
2283 assert_eq!(updated_tx.status, TransactionStatus::Failed);
2284 assert_eq!(
2285 updated_tx.status_reason.as_deref(),
2286 Some("Transaction reverted on-chain")
2287 );
2288 }
2289 }
2290
2291 mod handle_sent_state_tests {
2293 use super::*;
2294
2295 #[tokio::test]
2296 async fn test_sent_state_recent_no_resend() {
2297 let mut mocks = default_test_mocks();
2298 let relayer = create_test_relayer();
2299
2300 let mut tx = make_test_transaction(TransactionStatus::Sent);
2301 tx.sent_at = Some((Utc::now() - Duration::seconds(10)).to_rfc3339());
2303
2304 mocks
2306 .network_repo
2307 .expect_get_by_chain_id()
2308 .returning(|_, _| Ok(Some(create_test_network_model())));
2309
2310 mocks.provider.expect_get_block_by_number().returning(|| {
2312 Box::pin(async {
2313 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2314 let mut block: Block = Block::default();
2315 block.header.gas_limit = 30_000_000u64;
2316 Ok(AnyRpcBlock::from(block))
2317 })
2318 });
2319
2320 mocks
2322 .job_producer
2323 .expect_produce_check_transaction_status_job()
2324 .returning(|_, _| Box::pin(async { Ok(()) }));
2325
2326 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2327 let result = evm_transaction.handle_sent_state(tx.clone()).await.unwrap();
2328
2329 assert_eq!(result.status, TransactionStatus::Sent);
2330 }
2331
2332 #[tokio::test]
2333 async fn test_sent_state_stuck_schedules_resubmit() {
2334 let mut mocks = default_test_mocks();
2335 let relayer = create_test_relayer();
2336
2337 let mut tx = make_test_transaction(TransactionStatus::Sent);
2338 tx.sent_at = Some((Utc::now() - Duration::seconds(60)).to_rfc3339());
2340
2341 mocks
2343 .network_repo
2344 .expect_get_by_chain_id()
2345 .returning(|_, _| Ok(Some(create_test_network_model())));
2346
2347 mocks.provider.expect_get_block_by_number().returning(|| {
2349 Box::pin(async {
2350 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2351 let mut block: Block = Block::default();
2352 block.header.gas_limit = 30_000_000u64;
2353 Ok(AnyRpcBlock::from(block))
2354 })
2355 });
2356
2357 mocks
2359 .job_producer
2360 .expect_produce_submit_transaction_job()
2361 .returning(|_, _| Box::pin(async { Ok(()) }));
2362
2363 mocks
2365 .job_producer
2366 .expect_produce_check_transaction_status_job()
2367 .returning(|_, _| Box::pin(async { Ok(()) }));
2368
2369 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2370 let result = evm_transaction.handle_sent_state(tx.clone()).await.unwrap();
2371
2372 assert_eq!(result.status, TransactionStatus::Sent);
2373 }
2374 }
2375
2376 mod prepare_noop_update_request_tests {
2378 use super::*;
2379
2380 #[tokio::test]
2381 async fn test_noop_request_without_cancellation() {
2382 let mocks = default_test_mocks_with_network();
2384 let relayer = create_test_relayer();
2385 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2386 tx.noop_count = Some(2);
2387 tx.is_canceled = Some(false);
2388
2389 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2390 let update_req = evm_transaction
2391 .prepare_noop_update_request(&tx, false, None)
2392 .await
2393 .unwrap();
2394
2395 assert_eq!(update_req.noop_count, Some(3));
2397 assert_eq!(update_req.is_canceled, Some(false));
2399 }
2400
2401 #[tokio::test]
2402 async fn test_noop_request_with_cancellation() {
2403 let mocks = default_test_mocks_with_network();
2405 let relayer = create_test_relayer();
2406 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2407 tx.noop_count = None;
2408 tx.is_canceled = Some(false);
2409
2410 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2411 let update_req = evm_transaction
2412 .prepare_noop_update_request(&tx, true, None)
2413 .await
2414 .unwrap();
2415
2416 assert_eq!(update_req.noop_count, Some(1));
2418 assert_eq!(update_req.is_canceled, Some(true));
2420 }
2421 }
2422
2423 mod handle_submitted_state_tests {
2425 use super::*;
2426
2427 #[tokio::test]
2428 async fn test_schedules_resubmit_job() {
2429 let mut mocks = default_test_mocks();
2430 let relayer = create_test_relayer();
2431
2432 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2434 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2435
2436 mocks
2438 .network_repo
2439 .expect_get_by_chain_id()
2440 .returning(|_, _| Ok(Some(create_test_network_model())));
2441
2442 mocks.provider.expect_get_block_by_number().returning(|| {
2444 Box::pin(async {
2445 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2446 let mut block: Block = Block::default();
2447 block.header.gas_limit = 30_000_000u64;
2448 Ok(AnyRpcBlock::from(block))
2449 })
2450 });
2451
2452 mocks
2454 .provider
2455 .expect_get_transaction_count()
2456 .returning(|_| Box::pin(async { Ok(10) }));
2457
2458 mocks
2460 .job_producer
2461 .expect_produce_submit_transaction_job()
2462 .returning(|_, _| Box::pin(async { Ok(()) }));
2463
2464 mocks
2466 .job_producer
2467 .expect_produce_check_transaction_status_job()
2468 .returning(|_, _| Box::pin(async { Ok(()) }));
2469
2470 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2471 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2472
2473 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2475 }
2476
2477 #[tokio::test]
2480 async fn test_nonce_gap_detected_schedules_health_skips_resubmit() {
2481 let mut mocks = default_test_mocks();
2482 let relayer = create_test_relayer();
2483
2484 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2485 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2486 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
2487 nonce: Some(274),
2488 hash: Some("0xhash".to_string()),
2489 raw: Some(vec![1, 2, 3]),
2490 ..tx.network_data.get_evm_transaction_data().unwrap()
2491 });
2492
2493 mocks
2495 .network_repo
2496 .expect_get_by_chain_id()
2497 .returning(|_, _| Ok(Some(create_test_network_model())));
2498
2499 mocks
2501 .provider
2502 .expect_get_transaction_count()
2503 .returning(|_| Box::pin(async { Ok(269) }));
2504
2505 mocks
2507 .tx_repo
2508 .expect_get_nonce_occupancy()
2509 .withf(|relayer_id, from, to| {
2510 relayer_id == "test-relayer-id" && *from == 269 && *to == 274
2511 })
2512 .returning(|_, from, to| Ok((from..to).map(|n| (n, None)).collect()));
2513
2514 mocks
2516 .job_producer
2517 .expect_produce_relayer_health_check_job()
2518 .withf(|job, _| {
2519 job.metadata.as_ref().map_or(false, |m| {
2520 m.get("health_check_action") == Some(&"nonce_health".to_string())
2521 })
2522 })
2523 .returning(|_, _| Box::pin(async { Ok(()) }));
2524
2525 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2529 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2530
2531 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2532 }
2533
2534 #[tokio::test]
2537 async fn test_nonce_gap_check_rpc_failure_proceeds_to_resubmit() {
2538 let mut mocks = default_test_mocks();
2539 let relayer = create_test_relayer();
2540
2541 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2542 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2543
2544 mocks
2545 .network_repo
2546 .expect_get_by_chain_id()
2547 .returning(|_, _| Ok(Some(create_test_network_model())));
2548
2549 mocks.provider.expect_get_block_by_number().returning(|| {
2550 Box::pin(async {
2551 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2552 let mut block: Block = Block::default();
2553 block.header.gas_limit = 30_000_000u64;
2554 Ok(AnyRpcBlock::from(block))
2555 })
2556 });
2557
2558 mocks
2560 .provider
2561 .expect_get_transaction_count()
2562 .returning(|_| {
2563 Box::pin(async {
2564 Err(crate::services::provider::ProviderError::Other(
2565 "rpc timeout".to_string(),
2566 ))
2567 })
2568 });
2569
2570 mocks
2572 .job_producer
2573 .expect_produce_submit_transaction_job()
2574 .returning(|_, _| Box::pin(async { Ok(()) }));
2575
2576 mocks
2577 .job_producer
2578 .expect_produce_check_transaction_status_job()
2579 .returning(|_, _| Box::pin(async { Ok(()) }));
2580
2581 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2582 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2583
2584 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2585 }
2586
2587 #[tokio::test]
2590 async fn test_no_gap_when_slots_filled_proceeds_to_resubmit() {
2591 let mut mocks = default_test_mocks();
2592 let relayer = create_test_relayer();
2593
2594 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2595 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2596 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
2597 nonce: Some(270),
2598 hash: Some("0xhash".to_string()),
2599 raw: Some(vec![1, 2, 3]),
2600 ..tx.network_data.get_evm_transaction_data().unwrap()
2601 });
2602
2603 mocks
2604 .network_repo
2605 .expect_get_by_chain_id()
2606 .returning(|_, _| Ok(Some(create_test_network_model())));
2607
2608 mocks.provider.expect_get_block_by_number().returning(|| {
2609 Box::pin(async {
2610 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2611 let mut block: Block = Block::default();
2612 block.header.gas_limit = 30_000_000u64;
2613 Ok(AnyRpcBlock::from(block))
2614 })
2615 });
2616
2617 mocks
2619 .provider
2620 .expect_get_transaction_count()
2621 .returning(|_| Box::pin(async { Ok(269) }));
2622
2623 mocks
2625 .tx_repo
2626 .expect_get_nonce_occupancy()
2627 .returning(|_, from, to| {
2628 Ok((from..to)
2629 .map(|n| (n, Some(TransactionStatus::Submitted)))
2630 .collect())
2631 });
2632
2633 mocks
2635 .job_producer
2636 .expect_produce_submit_transaction_job()
2637 .returning(|_, _| Box::pin(async { Ok(()) }));
2638
2639 mocks
2640 .job_producer
2641 .expect_produce_check_transaction_status_job()
2642 .returning(|_, _| Box::pin(async { Ok(()) }));
2643
2644 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2645 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2646
2647 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2648 }
2649 }
2650
2651 mod handle_pending_state_tests {
2653 use super::*;
2654
2655 #[tokio::test]
2656 async fn test_pending_state_no_noop() {
2657 let mut mocks = default_test_mocks();
2659 let relayer = create_test_relayer();
2660 let mut tx = make_test_transaction(TransactionStatus::Pending);
2661 tx.created_at = Utc::now().to_rfc3339(); mocks
2665 .network_repo
2666 .expect_get_by_chain_id()
2667 .returning(|_, _| Ok(Some(create_test_network_model())));
2668
2669 mocks.provider.expect_get_block_by_number().returning(|| {
2671 Box::pin(async {
2672 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2673 let mut block: Block = Block::default();
2674 block.header.gas_limit = 30_000_000u64;
2675 Ok(AnyRpcBlock::from(block))
2676 })
2677 });
2678
2679 mocks
2681 .job_producer
2682 .expect_produce_check_transaction_status_job()
2683 .returning(|_, _| Box::pin(async { Ok(()) }));
2684
2685 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2686 let result = evm_transaction
2687 .handle_pending_state(tx.clone())
2688 .await
2689 .unwrap();
2690
2691 assert_eq!(result.id, tx.id);
2693 assert_eq!(result.status, tx.status);
2694 assert_eq!(result.noop_count, tx.noop_count);
2695 }
2696
2697 #[tokio::test]
2698 async fn test_pending_state_with_noop() {
2699 let mut mocks = default_test_mocks();
2701 let relayer = create_test_relayer();
2702 let mut tx = make_test_transaction(TransactionStatus::Pending);
2703 tx.created_at = (Utc::now() - Duration::minutes(2)).to_rfc3339();
2704
2705 mocks
2707 .network_repo
2708 .expect_get_by_chain_id()
2709 .returning(|_, _| Ok(Some(create_test_network_model())));
2710
2711 mocks.provider.expect_get_block_by_number().returning(|| {
2713 Box::pin(async {
2714 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2715 let mut block: Block = Block::default();
2716 block.header.gas_limit = 30_000_000u64;
2717 Ok(AnyRpcBlock::from(block))
2718 })
2719 });
2720
2721 let tx_clone = tx.clone();
2724 mocks
2725 .tx_repo
2726 .expect_partial_update()
2727 .withf(move |id, update| {
2728 id == "test-tx-id"
2729 && update.status == Some(TransactionStatus::Failed)
2730 && update.status_reason.is_some()
2731 })
2732 .returning(move |_, update| {
2733 let mut updated_tx = tx_clone.clone();
2734 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2735 updated_tx.status_reason = update.status_reason.clone();
2736 Ok(updated_tx)
2737 });
2738 mocks
2740 .job_producer
2741 .expect_produce_send_notification_job()
2742 .returning(|_, _| Box::pin(async { Ok(()) }));
2743
2744 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2745 let result = evm_transaction
2746 .handle_pending_state(tx.clone())
2747 .await
2748 .unwrap();
2749
2750 assert_eq!(result.status, TransactionStatus::Failed);
2752 assert!(result.status_reason.is_some());
2753 assert!(result.status_reason.unwrap().contains("Pending state"));
2754 }
2755 }
2756
2757 mod handle_mined_state_tests {
2759 use super::*;
2760
2761 #[tokio::test]
2762 async fn test_updates_status_and_schedules_check() {
2763 let mut mocks = default_test_mocks();
2764 let relayer = create_test_relayer();
2765 let tx = make_test_transaction(TransactionStatus::Submitted);
2767
2768 mocks
2770 .job_producer
2771 .expect_produce_check_transaction_status_job()
2772 .returning(|_, _| Box::pin(async { Ok(()) }));
2773 mocks
2775 .tx_repo
2776 .expect_partial_update()
2777 .returning(|_, update| {
2778 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2779 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2780 Ok(updated_tx)
2781 });
2782
2783 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2784 let result = evm_transaction
2785 .handle_mined_state(tx.clone())
2786 .await
2787 .unwrap();
2788 assert_eq!(result.status, TransactionStatus::Mined);
2789 }
2790 }
2791
2792 mod handle_final_state_tests {
2794 use super::*;
2795
2796 #[tokio::test]
2797 async fn test_final_state_confirmed() {
2798 let mut mocks = default_test_mocks();
2799 let relayer = create_test_relayer();
2800 let tx = make_test_transaction(TransactionStatus::Submitted);
2801
2802 mocks
2804 .tx_repo
2805 .expect_partial_update()
2806 .returning(|_, update| {
2807 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2808 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2809 Ok(updated_tx)
2810 });
2811
2812 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2813 let result = evm_transaction
2814 .handle_final_state(tx.clone(), TransactionStatus::Confirmed, None)
2815 .await
2816 .unwrap();
2817 assert_eq!(result.status, TransactionStatus::Confirmed);
2818 }
2819
2820 #[tokio::test]
2821 async fn test_final_state_failed() {
2822 let mut mocks = default_test_mocks();
2823 let relayer = create_test_relayer();
2824 let tx = make_test_transaction(TransactionStatus::Submitted);
2825
2826 mocks
2828 .tx_repo
2829 .expect_partial_update()
2830 .returning(|_, update| {
2831 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2832 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2833 updated_tx.status_reason = update.status_reason.clone();
2834 Ok(updated_tx)
2835 });
2836
2837 let reason = "Transaction reverted on-chain (receipt status: failed)".to_string();
2838 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2839 let result = evm_transaction
2840 .handle_final_state(tx.clone(), TransactionStatus::Failed, Some(reason.clone()))
2841 .await
2842 .unwrap();
2843 assert_eq!(result.status, TransactionStatus::Failed);
2844 assert_eq!(result.status_reason.as_deref(), Some(reason.as_str()));
2845 }
2846
2847 #[tokio::test]
2848 async fn test_final_state_expired() {
2849 let mut mocks = default_test_mocks();
2850 let relayer = create_test_relayer();
2851 let tx = make_test_transaction(TransactionStatus::Submitted);
2852
2853 mocks
2855 .tx_repo
2856 .expect_partial_update()
2857 .returning(|_, update| {
2858 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2859 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2860 Ok(updated_tx)
2861 });
2862
2863 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2864 let result = evm_transaction
2865 .handle_final_state(tx.clone(), TransactionStatus::Expired, None)
2866 .await
2867 .unwrap();
2868 assert_eq!(result.status, TransactionStatus::Expired);
2869 }
2870 }
2871
2872 mod handle_status_impl_tests {
2874 use super::*;
2875
2876 #[tokio::test]
2877 async fn test_impl_submitted_branch() {
2878 let mut mocks = default_test_mocks();
2879 let relayer = create_test_relayer();
2880 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2881 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
2882 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2884 evm_data.hash = Some("0xFakeHash".to_string());
2885 }
2886 mocks
2888 .provider
2889 .expect_get_transaction_receipt()
2890 .returning(|_| Box::pin(async { Ok(None) }));
2891 mocks
2893 .network_repo
2894 .expect_get_by_chain_id()
2895 .returning(|_, _| Ok(Some(create_test_network_model())));
2896 mocks
2898 .job_producer
2899 .expect_produce_check_transaction_status_job()
2900 .returning(|_, _| Box::pin(async { Ok(()) }));
2901 mocks
2903 .tx_repo
2904 .expect_partial_update()
2905 .returning(|_, update| {
2906 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2907 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2908 Ok(updated_tx)
2909 });
2910
2911 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2912 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2913 assert_eq!(result.status, TransactionStatus::Submitted);
2914 }
2915
2916 #[tokio::test]
2917 async fn test_impl_mined_branch() {
2918 let mut mocks = default_test_mocks();
2919 let relayer = create_test_relayer();
2920 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2921 tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
2923 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2925 evm_data.hash = Some("0xFakeHash".to_string());
2926 }
2927 mocks
2929 .provider
2930 .expect_get_transaction_receipt()
2931 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
2932 mocks
2934 .provider
2935 .expect_get_block_number()
2936 .return_once(|| Box::pin(async { Ok(100) }));
2937 mocks
2939 .network_repo
2940 .expect_get_by_chain_id()
2941 .returning(|_, _| Ok(Some(create_test_network_model())));
2942 mocks
2944 .job_producer
2945 .expect_produce_send_notification_job()
2946 .returning(|_, _| Box::pin(async { Ok(()) }));
2947 mocks.tx_repo.expect_get_by_id().returning(|_| {
2949 let updated_tx = make_test_transaction(TransactionStatus::Mined);
2950 Ok(updated_tx)
2951 });
2952 mocks
2954 .tx_repo
2955 .expect_partial_update()
2956 .returning(|_, update| {
2957 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2958 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2959 Ok(updated_tx)
2960 });
2961
2962 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2963 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2964 assert_eq!(result.status, TransactionStatus::Mined);
2965 }
2966
2967 #[tokio::test]
2968 async fn test_impl_final_confirmed_branch() {
2969 let mut mocks = default_test_mocks();
2970 let relayer = create_test_relayer();
2971 let tx = make_test_transaction(TransactionStatus::Confirmed);
2973
2974 mocks
2977 .tx_repo
2978 .expect_partial_update()
2979 .returning(|_, update| {
2980 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2981 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2982 Ok(updated_tx)
2983 });
2984
2985 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2986 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2987 assert_eq!(result.status, TransactionStatus::Confirmed);
2988 }
2989
2990 #[tokio::test]
2991 async fn test_impl_final_failed_branch() {
2992 let mut mocks = default_test_mocks();
2993 let relayer = create_test_relayer();
2994 let tx = make_test_transaction(TransactionStatus::Failed);
2996
2997 mocks
2998 .tx_repo
2999 .expect_partial_update()
3000 .returning(|_, update| {
3001 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3002 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3003 Ok(updated_tx)
3004 });
3005
3006 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3007 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3008 assert_eq!(result.status, TransactionStatus::Failed);
3009 }
3010
3011 #[tokio::test]
3014 async fn test_impl_submitted_to_failed_sets_status_reason() {
3015 let mut mocks = default_test_mocks();
3016 let relayer = create_test_relayer();
3017 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3018 tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
3019 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3020 evm_data.hash = Some("0xFakeHash".to_string());
3021 }
3022
3023 mocks
3025 .provider
3026 .expect_get_transaction_receipt()
3027 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
3028
3029 mocks
3033 .provider
3034 .expect_raw_request_dyn()
3035 .returning(|_, _| Box::pin(async { Ok(serde_json::json!({})) }));
3036
3037 let tx_clone = tx.clone();
3039 mocks.tx_repo.expect_get_by_id().returning(move |_| {
3040 let mut reloaded = tx_clone.clone();
3041 reloaded.status = TransactionStatus::Submitted;
3042 Ok(reloaded)
3043 });
3044
3045 mocks
3047 .tx_repo
3048 .expect_partial_update()
3049 .withf(|_, update| {
3050 update.status == Some(TransactionStatus::Failed)
3051 && update.status_reason.is_some()
3052 })
3053 .returning(|_, update| {
3054 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3055 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3056 updated_tx.status_reason = update.status_reason.clone();
3057 Ok(updated_tx)
3058 });
3059
3060 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3061 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3062 assert_eq!(result.status, TransactionStatus::Failed);
3063 assert!(result.status_reason.is_some());
3064 assert!(
3065 result
3066 .status_reason
3067 .as_ref()
3068 .unwrap()
3069 .contains("reverted on-chain"),
3070 "Expected on-chain revert reason, got: {:?}",
3071 result.status_reason
3072 );
3073 }
3074
3075 #[tokio::test]
3076 async fn test_impl_final_expired_branch() {
3077 let mut mocks = default_test_mocks();
3078 let relayer = create_test_relayer();
3079 let tx = make_test_transaction(TransactionStatus::Expired);
3081
3082 mocks
3083 .tx_repo
3084 .expect_partial_update()
3085 .returning(|_, update| {
3086 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3087 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3088 Ok(updated_tx)
3089 });
3090
3091 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3092 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3093 assert_eq!(result.status, TransactionStatus::Expired);
3094 }
3095 }
3096
3097 mod revert_data_recovery_tests {
3099 use super::*;
3100 use crate::services::provider::ProviderError;
3101 use alloy::primitives::Bytes;
3102
3103 const LEGACY_REASON: &str = "Transaction reverted on-chain (receipt status: failed)";
3104
3105 fn failing_submitted_tx() -> TransactionRepoModel {
3108 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3109 tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
3110 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3111 evm_data.from = "0x0000000000000000000000000000000000000001".to_string();
3112 evm_data.to = Some("0x0000000000000000000000000000000000000002".to_string());
3113 evm_data.data = Some("0xabcdef".to_string());
3114 evm_data.hash = Some("0xFakeHash".to_string());
3115 }
3116 tx
3117 }
3118
3119 fn expect_failed_receipt(mocks: &mut TestMocks) {
3122 mocks
3123 .provider
3124 .expect_get_transaction_receipt()
3125 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
3126 }
3127
3128 fn expect_reload_and_capture(mocks: &mut TestMocks) {
3131 mocks.tx_repo.expect_get_by_id().returning(|_| {
3132 let mut reloaded = failing_submitted_tx();
3133 reloaded.status = TransactionStatus::Submitted;
3134 Ok(reloaded)
3135 });
3136 mocks
3137 .tx_repo
3138 .expect_partial_update()
3139 .returning(|_, update| {
3140 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3141 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3142 updated_tx.status_reason = update.status_reason.clone();
3143 Ok(updated_tx)
3144 });
3145 }
3146
3147 #[tokio::test]
3149 async fn test_eth_call_revert_data_enriches_status_reason() {
3150 let mut mocks = default_test_mocks();
3151 let relayer = create_test_relayer();
3152 let tx = failing_submitted_tx();
3153
3154 expect_failed_receipt(&mut mocks);
3155 mocks
3157 .provider
3158 .expect_raw_request_dyn()
3159 .returning(|_, _| Box::pin(async { Ok(serde_json::json!({})) }));
3160 mocks
3161 .provider
3162 .expect_get_call_revert_data()
3163 .returning(|_, _| {
3164 Box::pin(async { Ok(Some(Bytes::from(hex::decode("5592f1b2").unwrap()))) })
3165 });
3166 expect_reload_and_capture(&mut mocks);
3167
3168 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3169 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3170 assert_eq!(result.status, TransactionStatus::Failed);
3171 let reason = result.status_reason.unwrap();
3172 assert!(
3173 reason.contains("revert_data: 0x5592f1b2"),
3174 "expected enriched revert_data, got: {reason}"
3175 );
3176 }
3177
3178 #[tokio::test]
3180 async fn test_no_revert_data_uses_legacy_reason() {
3181 let mut mocks = default_test_mocks();
3182 let relayer = create_test_relayer();
3183 let tx = failing_submitted_tx();
3184
3185 expect_failed_receipt(&mut mocks);
3186 mocks
3187 .provider
3188 .expect_raw_request_dyn()
3189 .returning(|_, _| Box::pin(async { Ok(serde_json::json!({})) }));
3190 mocks
3191 .provider
3192 .expect_get_call_revert_data()
3193 .returning(|_, _| Box::pin(async { Ok(None) }));
3194 expect_reload_and_capture(&mut mocks);
3195
3196 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3197 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3198 assert_eq!(result.status, TransactionStatus::Failed);
3199 assert_eq!(result.status_reason.unwrap(), LEGACY_REASON);
3200 }
3201
3202 #[tokio::test]
3204 async fn test_already_failed_skips_recovery() {
3205 let mocks = default_test_mocks();
3207 let relayer = create_test_relayer();
3208 let tx = make_test_transaction(TransactionStatus::Failed);
3209
3210 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3211 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3212 assert_eq!(result.status, TransactionStatus::Failed);
3213 assert!(
3214 result.status_reason.is_none(),
3215 "status_reason should not be rewritten on re-poll, got: {:?}",
3216 result.status_reason
3217 );
3218 }
3219
3220 #[tokio::test]
3222 async fn test_trace_output_used_without_eth_call() {
3223 let mut mocks = default_test_mocks();
3224 let relayer = create_test_relayer();
3225 let tx = failing_submitted_tx();
3226
3227 expect_failed_receipt(&mut mocks);
3228 mocks.provider.expect_raw_request_dyn().returning(|_, _| {
3229 Box::pin(async { Ok(serde_json::json!({"output": "0xdeadbeef"})) })
3230 });
3231 mocks.provider.expect_get_call_revert_data().never();
3233 expect_reload_and_capture(&mut mocks);
3234
3235 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3236 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3237 assert_eq!(result.status, TransactionStatus::Failed);
3238 assert!(result
3239 .status_reason
3240 .unwrap()
3241 .contains("revert_data: 0xdeadbeef"));
3242 }
3243
3244 #[tokio::test]
3246 async fn test_trace_unavailable_falls_back_to_eth_call() {
3247 let mut mocks = default_test_mocks();
3248 let relayer = create_test_relayer();
3249 let tx = failing_submitted_tx();
3250
3251 expect_failed_receipt(&mut mocks);
3252 mocks.provider.expect_raw_request_dyn().returning(|_, _| {
3253 Box::pin(async { Err(ProviderError::Other("method not supported".to_string())) })
3254 });
3255 mocks
3256 .provider
3257 .expect_get_call_revert_data()
3258 .withf(|req, _block| req.gas_price == Some(20000000000))
3261 .returning(|_, _| {
3262 Box::pin(async { Ok(Some(Bytes::from(hex::decode("08c379a0").unwrap()))) })
3263 });
3264 expect_reload_and_capture(&mut mocks);
3265
3266 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3267 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3268 assert_eq!(result.status, TransactionStatus::Failed);
3269 assert!(result
3270 .status_reason
3271 .unwrap()
3272 .contains("revert_data: 0x08c379a0"));
3273 }
3274
3275 #[tokio::test]
3277 async fn test_both_paths_fail_uses_legacy_reason() {
3278 let mut mocks = default_test_mocks();
3279 let relayer = create_test_relayer();
3280 let tx = failing_submitted_tx();
3281
3282 expect_failed_receipt(&mut mocks);
3283 mocks.provider.expect_raw_request_dyn().returning(|_, _| {
3284 Box::pin(async { Err(ProviderError::Other("trace down".to_string())) })
3285 });
3286 mocks
3287 .provider
3288 .expect_get_call_revert_data()
3289 .returning(|_, _| {
3290 Box::pin(async { Err(ProviderError::Other("eth_call down".to_string())) })
3291 });
3292 expect_reload_and_capture(&mut mocks);
3293
3294 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3295 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3296 assert_eq!(result.status, TransactionStatus::Failed);
3297 assert_eq!(result.status_reason.unwrap(), LEGACY_REASON);
3298 }
3299
3300 fn relayer_with_revert_policy(include: Option<bool>) -> RelayerRepoModel {
3302 let mut relayer = create_test_relayer();
3303 relayer.policies = RelayerNetworkPolicy::Evm(RelayerEvmPolicy {
3304 include_revert_data: include,
3305 ..Default::default()
3306 });
3307 relayer
3308 }
3309
3310 #[tokio::test]
3312 async fn test_recovery_disabled_issues_no_recovery_rpcs() {
3313 let mut mocks = default_test_mocks();
3314 let relayer = relayer_with_revert_policy(Some(false));
3315 let tx = failing_submitted_tx();
3316
3317 expect_failed_receipt(&mut mocks);
3319 mocks.provider.expect_raw_request_dyn().never();
3320 mocks.provider.expect_get_call_revert_data().never();
3321 expect_reload_and_capture(&mut mocks);
3322
3323 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3324 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3325 assert_eq!(result.status, TransactionStatus::Failed);
3326 assert_eq!(result.status_reason.unwrap(), LEGACY_REASON);
3327 }
3328
3329 #[tokio::test]
3331 async fn test_recovery_attempted_when_enabled() {
3332 for include in [None, Some(true)] {
3333 let mut mocks = default_test_mocks();
3334 let relayer = relayer_with_revert_policy(include);
3335 let tx = failing_submitted_tx();
3336
3337 expect_failed_receipt(&mut mocks);
3338 mocks
3340 .provider
3341 .expect_raw_request_dyn()
3342 .times(1)
3343 .returning(|_, _| {
3344 Box::pin(async { Ok(serde_json::json!({"output": "0xfeed"})) })
3345 });
3346 expect_reload_and_capture(&mut mocks);
3347
3348 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3349 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3350 assert_eq!(result.status, TransactionStatus::Failed);
3351 assert!(
3352 result
3353 .status_reason
3354 .unwrap()
3355 .contains("revert_data: 0xfeed"),
3356 "recovery should run for include_revert_data = {include:?}"
3357 );
3358 }
3359 }
3360 }
3361
3362 mod circuit_breaker_tests {
3364 use super::*;
3365 use crate::jobs::StatusCheckContext;
3366
3367 fn create_triggered_context() -> StatusCheckContext {
3369 StatusCheckContext::new(
3370 30, 50, 60, 25, 75, NetworkType::Evm,
3376 )
3377 }
3378
3379 fn create_safe_context() -> StatusCheckContext {
3381 StatusCheckContext::new(
3382 5, 10, 15, 25, 75, NetworkType::Evm,
3388 )
3389 }
3390
3391 fn create_total_triggered_context() -> StatusCheckContext {
3393 StatusCheckContext::new(
3394 5, 80, 100, 25, 75, NetworkType::Evm,
3400 )
3401 }
3402
3403 #[tokio::test]
3404 async fn test_circuit_breaker_pending_marks_as_failed() {
3405 let mut mocks = default_test_mocks();
3406 let relayer = create_test_relayer();
3407 let tx = make_test_transaction(TransactionStatus::Pending);
3408
3409 mocks
3411 .tx_repo
3412 .expect_partial_update()
3413 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3414 .returning(|_, update| {
3415 let mut updated_tx = make_test_transaction(TransactionStatus::Pending);
3416 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3417 updated_tx.status_reason = update.status_reason.clone();
3418 Ok(updated_tx)
3419 });
3420
3421 mocks
3423 .job_producer
3424 .expect_produce_send_notification_job()
3425 .returning(|_, _| Box::pin(async { Ok(()) }));
3426
3427 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3428 let ctx = create_triggered_context();
3429
3430 let result = evm_transaction
3431 .handle_status_impl(tx, Some(ctx))
3432 .await
3433 .unwrap();
3434
3435 assert_eq!(result.status, TransactionStatus::Failed);
3436 assert!(result.status_reason.is_some());
3437 assert!(result.status_reason.unwrap().contains("consecutive errors"));
3438 }
3439
3440 #[tokio::test]
3441 async fn test_circuit_breaker_sent_marks_as_failed() {
3442 let mut mocks = default_test_mocks();
3443 let relayer = create_test_relayer();
3444 let tx = make_test_transaction(TransactionStatus::Sent);
3445
3446 mocks
3448 .tx_repo
3449 .expect_partial_update()
3450 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3451 .returning(|_, update| {
3452 let mut updated_tx = make_test_transaction(TransactionStatus::Sent);
3453 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3454 updated_tx.status_reason = update.status_reason.clone();
3455 Ok(updated_tx)
3456 });
3457
3458 mocks
3460 .job_producer
3461 .expect_produce_send_notification_job()
3462 .returning(|_, _| Box::pin(async { Ok(()) }));
3463
3464 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3465 let ctx = create_triggered_context();
3466
3467 let result = evm_transaction
3468 .handle_status_impl(tx, Some(ctx))
3469 .await
3470 .unwrap();
3471
3472 assert_eq!(result.status, TransactionStatus::Failed);
3473 }
3474
3475 #[tokio::test]
3476 async fn test_circuit_breaker_submitted_triggers_noop() {
3477 let mut mocks = default_test_mocks();
3478 let relayer = create_test_relayer();
3479 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3480 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3481
3482 mocks
3484 .network_repo
3485 .expect_get_by_chain_id()
3486 .returning(|_, _| Ok(Some(create_test_network_model())));
3487
3488 mocks
3490 .tx_repo
3491 .expect_partial_update()
3492 .returning(|_, update| {
3493 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3494 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3495 updated_tx.status_reason = update.status_reason.clone();
3496 updated_tx.noop_count = update.noop_count;
3497 Ok(updated_tx)
3498 });
3499
3500 mocks
3502 .job_producer
3503 .expect_produce_submit_transaction_job()
3504 .returning(|_, _| Box::pin(async { Ok(()) }));
3505
3506 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3507 let ctx = create_triggered_context();
3508
3509 let result = evm_transaction
3510 .handle_status_impl(tx, Some(ctx))
3511 .await
3512 .unwrap();
3513
3514 assert!(result.noop_count.is_some());
3516 }
3517
3518 #[tokio::test]
3519 async fn test_circuit_breaker_noop_tx_excluded() {
3520 let mut mocks = default_test_mocks();
3521 let relayer = create_test_relayer();
3522
3523 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3525 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3526 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3527 evm_data.to = Some(evm_data.from.clone()); evm_data.value = U256::from(0);
3529 evm_data.data = Some("0x".to_string());
3530 evm_data.hash = Some("0xFakeHash".to_string());
3531 }
3532
3533 mocks
3536 .provider
3537 .expect_get_transaction_receipt()
3538 .returning(|_| Box::pin(async { Ok(None) }));
3539
3540 mocks
3541 .network_repo
3542 .expect_get_by_chain_id()
3543 .returning(|_, _| Ok(Some(create_test_network_model())));
3544
3545 mocks
3546 .job_producer
3547 .expect_produce_check_transaction_status_job()
3548 .returning(|_, _| Box::pin(async { Ok(()) }));
3549
3550 mocks
3552 .job_producer
3553 .expect_produce_submit_transaction_job()
3554 .returning(|_, _| Box::pin(async { Ok(()) }));
3555
3556 mocks
3557 .tx_repo
3558 .expect_partial_update()
3559 .returning(|_, update| {
3560 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3561 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3562 Ok(updated_tx)
3563 });
3564
3565 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3566 let ctx = create_triggered_context();
3567
3568 let result = evm_transaction
3569 .handle_status_impl(tx, Some(ctx))
3570 .await
3571 .unwrap();
3572
3573 assert_eq!(result.status, TransactionStatus::Submitted);
3575 }
3576
3577 #[tokio::test]
3578 async fn test_circuit_breaker_total_failures_triggers() {
3579 let mut mocks = default_test_mocks();
3580 let relayer = create_test_relayer();
3581 let tx = make_test_transaction(TransactionStatus::Pending);
3582
3583 mocks
3585 .tx_repo
3586 .expect_partial_update()
3587 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3588 .returning(|_, update| {
3589 let mut updated_tx = make_test_transaction(TransactionStatus::Pending);
3590 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3591 updated_tx.status_reason = update.status_reason.clone();
3592 Ok(updated_tx)
3593 });
3594
3595 mocks
3596 .job_producer
3597 .expect_produce_send_notification_job()
3598 .returning(|_, _| Box::pin(async { Ok(()) }));
3599
3600 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3601 let ctx = create_total_triggered_context();
3603
3604 let result = evm_transaction
3605 .handle_status_impl(tx, Some(ctx))
3606 .await
3607 .unwrap();
3608
3609 assert_eq!(result.status, TransactionStatus::Failed);
3610 assert!(result.status_reason.is_some());
3611 }
3612
3613 #[tokio::test]
3614 async fn test_circuit_breaker_below_threshold_continues_normally() {
3615 let mut mocks = default_test_mocks();
3616 let relayer = create_test_relayer();
3617 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3618 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3619
3620 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3621 evm_data.hash = Some("0xFakeHash".to_string());
3622 }
3623
3624 mocks
3626 .provider
3627 .expect_get_transaction_receipt()
3628 .returning(|_| Box::pin(async { Ok(None) }));
3629
3630 mocks
3631 .network_repo
3632 .expect_get_by_chain_id()
3633 .returning(|_, _| Ok(Some(create_test_network_model())));
3634
3635 mocks
3636 .job_producer
3637 .expect_produce_check_transaction_status_job()
3638 .returning(|_, _| Box::pin(async { Ok(()) }));
3639
3640 mocks
3641 .tx_repo
3642 .expect_partial_update()
3643 .returning(|_, update| {
3644 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3645 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3646 Ok(updated_tx)
3647 });
3648
3649 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3650 let ctx = create_safe_context();
3651
3652 let result = evm_transaction
3653 .handle_status_impl(tx, Some(ctx))
3654 .await
3655 .unwrap();
3656
3657 assert_eq!(result.status, TransactionStatus::Submitted);
3659 }
3660
3661 #[tokio::test]
3662 async fn test_circuit_breaker_no_context_continues_normally() {
3663 let mut mocks = default_test_mocks();
3664 let relayer = create_test_relayer();
3665 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3666 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3667
3668 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3669 evm_data.hash = Some("0xFakeHash".to_string());
3670 }
3671
3672 mocks
3674 .provider
3675 .expect_get_transaction_receipt()
3676 .returning(|_| Box::pin(async { Ok(None) }));
3677
3678 mocks
3679 .network_repo
3680 .expect_get_by_chain_id()
3681 .returning(|_, _| Ok(Some(create_test_network_model())));
3682
3683 mocks
3684 .job_producer
3685 .expect_produce_check_transaction_status_job()
3686 .returning(|_, _| Box::pin(async { Ok(()) }));
3687
3688 mocks
3689 .tx_repo
3690 .expect_partial_update()
3691 .returning(|_, update| {
3692 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3693 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3694 Ok(updated_tx)
3695 });
3696
3697 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3698
3699 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3701
3702 assert_eq!(result.status, TransactionStatus::Submitted);
3704 }
3705
3706 #[tokio::test]
3707 async fn test_circuit_breaker_final_state_early_return() {
3708 let mocks = default_test_mocks();
3709 let relayer = create_test_relayer();
3710 let tx = make_test_transaction(TransactionStatus::Confirmed);
3712
3713 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3714 let ctx = create_triggered_context();
3715
3716 let result = evm_transaction
3718 .handle_status_impl(tx, Some(ctx))
3719 .await
3720 .unwrap();
3721
3722 assert_eq!(result.status, TransactionStatus::Confirmed);
3723 }
3724 }
3725
3726 mod hash_recovery_tests {
3728 use super::*;
3729
3730 #[tokio::test]
3731 async fn test_should_try_hash_recovery_not_submitted() {
3732 let mocks = default_test_mocks();
3733 let relayer = create_test_relayer();
3734
3735 let mut tx = make_test_transaction(TransactionStatus::Sent);
3736 tx.hashes = vec![
3737 "0xHash1".to_string(),
3738 "0xHash2".to_string(),
3739 "0xHash3".to_string(),
3740 ];
3741
3742 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3743 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3744
3745 assert!(
3746 !result,
3747 "Should not attempt recovery for non-Submitted transactions"
3748 );
3749 }
3750
3751 #[tokio::test]
3752 async fn test_should_try_hash_recovery_not_enough_hashes() {
3753 let mocks = default_test_mocks();
3754 let relayer = create_test_relayer();
3755
3756 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3757 tx.hashes = vec!["0xHash1".to_string()]; tx.sent_at = Some((Utc::now() - Duration::minutes(3)).to_rfc3339());
3759
3760 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3761 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3762
3763 assert!(
3764 !result,
3765 "Should not attempt recovery with insufficient hashes"
3766 );
3767 }
3768
3769 #[tokio::test]
3770 async fn test_should_try_hash_recovery_too_recent() {
3771 let mocks = default_test_mocks();
3772 let relayer = create_test_relayer();
3773
3774 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3775 tx.hashes = vec![
3776 "0xHash1".to_string(),
3777 "0xHash2".to_string(),
3778 "0xHash3".to_string(),
3779 ];
3780 tx.sent_at = Some(Utc::now().to_rfc3339()); let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3783 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3784
3785 assert!(
3786 !result,
3787 "Should not attempt recovery for recently sent transactions"
3788 );
3789 }
3790
3791 #[tokio::test]
3792 async fn test_should_try_hash_recovery_success() {
3793 let mocks = default_test_mocks();
3794 let relayer = create_test_relayer();
3795
3796 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3797 tx.hashes = vec![
3798 "0xHash1".to_string(),
3799 "0xHash2".to_string(),
3800 "0xHash3".to_string(),
3801 ];
3802 tx.sent_at = Some((Utc::now() - Duration::minutes(3)).to_rfc3339());
3803
3804 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3805 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3806
3807 assert!(
3808 result,
3809 "Should attempt recovery for stuck transactions with multiple hashes"
3810 );
3811 }
3812
3813 #[tokio::test]
3814 async fn test_try_recover_no_historical_hash_found() {
3815 let mut mocks = default_test_mocks();
3816 let relayer = create_test_relayer();
3817
3818 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3819 tx.hashes = vec![
3820 "0xHash1".to_string(),
3821 "0xHash2".to_string(),
3822 "0xHash3".to_string(),
3823 ];
3824
3825 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3826 evm_data.hash = Some("0xHash3".to_string());
3827 }
3828
3829 mocks
3831 .provider
3832 .expect_get_transaction_receipt()
3833 .returning(|_| Box::pin(async { Ok(None) }));
3834
3835 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3836 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3837 let result = evm_transaction
3838 .try_recover_with_historical_hashes(&tx, &evm_data)
3839 .await
3840 .unwrap();
3841
3842 assert!(
3843 result.is_none(),
3844 "Should return None when no historical hash is found"
3845 );
3846 }
3847
3848 #[tokio::test]
3849 async fn test_try_recover_finds_mined_historical_hash() {
3850 let mut mocks = default_test_mocks();
3851 let relayer = create_test_relayer();
3852
3853 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3854 tx.hashes = vec![
3855 "0xHash1".to_string(),
3856 "0xHash2".to_string(), "0xHash3".to_string(),
3858 ];
3859
3860 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3861 evm_data.hash = Some("0xHash3".to_string()); }
3863
3864 mocks
3866 .provider
3867 .expect_get_transaction_receipt()
3868 .returning(|hash| {
3869 if hash == "0xHash2" {
3870 Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) })
3871 } else {
3872 Box::pin(async { Ok(None) })
3873 }
3874 });
3875
3876 let tx_clone = tx.clone();
3878 mocks
3879 .tx_repo
3880 .expect_partial_update()
3881 .returning(move |_, update| {
3882 let mut updated_tx = tx_clone.clone();
3883 if let Some(status) = update.status {
3884 updated_tx.status = status;
3885 }
3886 if let Some(NetworkTransactionData::Evm(ref evm_data)) = update.network_data {
3887 if let NetworkTransactionData::Evm(ref mut updated_evm) =
3888 updated_tx.network_data
3889 {
3890 updated_evm.hash = evm_data.hash.clone();
3891 }
3892 }
3893 Ok(updated_tx)
3894 });
3895
3896 mocks
3898 .job_producer
3899 .expect_produce_send_notification_job()
3900 .returning(|_, _| Box::pin(async { Ok(()) }));
3901
3902 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3903 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3904 let result = evm_transaction
3905 .try_recover_with_historical_hashes(&tx, &evm_data)
3906 .await
3907 .unwrap();
3908
3909 assert!(result.is_some(), "Should recover the transaction");
3910 let recovered_tx = result.unwrap();
3911 assert_eq!(recovered_tx.status, TransactionStatus::Mined);
3912 }
3913
3914 #[tokio::test]
3915 async fn test_try_recover_network_error_continues() {
3916 let mut mocks = default_test_mocks();
3917 let relayer = create_test_relayer();
3918
3919 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3920 tx.hashes = vec![
3921 "0xHash1".to_string(),
3922 "0xHash2".to_string(), "0xHash3".to_string(), ];
3925
3926 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3927 evm_data.hash = Some("0xHash1".to_string());
3928 }
3929
3930 mocks
3932 .provider
3933 .expect_get_transaction_receipt()
3934 .returning(|hash| {
3935 if hash == "0xHash2" {
3936 Box::pin(async { Err(crate::services::provider::ProviderError::Timeout) })
3937 } else if hash == "0xHash3" {
3938 Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) })
3939 } else {
3940 Box::pin(async { Ok(None) })
3941 }
3942 });
3943
3944 let tx_clone = tx.clone();
3946 mocks
3947 .tx_repo
3948 .expect_partial_update()
3949 .returning(move |_, update| {
3950 let mut updated_tx = tx_clone.clone();
3951 if let Some(status) = update.status {
3952 updated_tx.status = status;
3953 }
3954 Ok(updated_tx)
3955 });
3956
3957 mocks
3959 .job_producer
3960 .expect_produce_send_notification_job()
3961 .returning(|_, _| Box::pin(async { Ok(()) }));
3962
3963 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3964 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3965 let result = evm_transaction
3966 .try_recover_with_historical_hashes(&tx, &evm_data)
3967 .await
3968 .unwrap();
3969
3970 assert!(
3971 result.is_some(),
3972 "Should continue checking after network error and find mined hash"
3973 );
3974 }
3975
3976 #[tokio::test]
3977 async fn test_update_transaction_with_corrected_hash() {
3978 let mut mocks = default_test_mocks();
3979 let relayer = create_test_relayer();
3980
3981 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3982 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3983 evm_data.hash = Some("0xWrongHash".to_string());
3984 }
3985
3986 mocks
3988 .tx_repo
3989 .expect_partial_update()
3990 .returning(move |_, update| {
3991 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3992 if let Some(status) = update.status {
3993 updated_tx.status = status;
3994 }
3995 if let Some(NetworkTransactionData::Evm(ref evm_data)) = update.network_data {
3996 if let NetworkTransactionData::Evm(ref mut updated_evm) =
3997 updated_tx.network_data
3998 {
3999 updated_evm.hash = evm_data.hash.clone();
4000 }
4001 }
4002 Ok(updated_tx)
4003 });
4004
4005 mocks
4007 .job_producer
4008 .expect_produce_send_notification_job()
4009 .returning(|_, _| Box::pin(async { Ok(()) }));
4010
4011 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4012 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
4013 let result = evm_transaction
4014 .update_transaction_with_corrected_hash(
4015 &tx,
4016 &evm_data,
4017 "0xCorrectHash",
4018 TransactionStatus::Mined,
4019 )
4020 .await
4021 .unwrap();
4022
4023 assert_eq!(result.status, TransactionStatus::Mined);
4024 if let NetworkTransactionData::Evm(ref updated_evm) = result.network_data {
4025 assert_eq!(updated_evm.hash.as_ref().unwrap(), "0xCorrectHash");
4026 }
4027 }
4028 }
4029
4030 mod check_transaction_status_edge_cases {
4032 use super::*;
4033
4034 #[tokio::test]
4035 async fn test_missing_hash_returns_error() {
4036 let mocks = default_test_mocks();
4037 let relayer = create_test_relayer();
4038
4039 let tx = make_test_transaction(TransactionStatus::Submitted);
4040 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4043 let result = evm_transaction.check_transaction_status(&tx).await;
4044
4045 assert!(result.is_err(), "Should return error when hash is missing");
4046 }
4047
4048 #[tokio::test]
4049 async fn test_pending_status_early_return() {
4050 let mocks = default_test_mocks();
4051 let relayer = create_test_relayer();
4052
4053 let tx = make_test_transaction(TransactionStatus::Pending);
4054
4055 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4056 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
4057
4058 assert_eq!(
4059 status,
4060 TransactionStatus::Pending,
4061 "Should return Pending without querying blockchain"
4062 );
4063 }
4064
4065 #[tokio::test]
4066 async fn test_sent_status_early_return() {
4067 let mocks = default_test_mocks();
4068 let relayer = create_test_relayer();
4069
4070 let tx = make_test_transaction(TransactionStatus::Sent);
4071
4072 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4073 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
4074
4075 assert_eq!(
4076 status,
4077 TransactionStatus::Sent,
4078 "Should return Sent without querying blockchain"
4079 );
4080 }
4081
4082 #[tokio::test]
4083 async fn test_final_state_early_return() {
4084 let mocks = default_test_mocks();
4085 let relayer = create_test_relayer();
4086
4087 let tx = make_test_transaction(TransactionStatus::Confirmed);
4088
4089 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4090 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
4091
4092 assert_eq!(
4093 status,
4094 TransactionStatus::Confirmed,
4095 "Should return final state without querying blockchain"
4096 );
4097 }
4098 }
4099
4100 mod nonce_recovery_tests {
4101 use super::*;
4102 use crate::domain::transaction::evm::evm_transaction::TX_NONCE_RECONCILE_TRIGGER;
4103 use crate::jobs::StatusCheckContext;
4104
4105 #[tokio::test]
4107 async fn test_nonce_recovery_nonce_consumed_externally() {
4108 let mut mocks = default_test_mocks();
4109 let relayer = create_test_relayer();
4110
4111 let mut tx = make_test_transaction(TransactionStatus::Submitted);
4112 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4113 nonce: Some(5),
4114 hash: Some("0xhash".to_string()),
4115 raw: Some(vec![1, 2, 3]),
4116 ..tx.network_data.get_evm_transaction_data().unwrap()
4117 });
4118 tx.sent_at = Some(Utc::now().to_rfc3339());
4119
4120 mocks
4122 .provider
4123 .expect_get_transaction_receipt()
4124 .returning(|_| Box::pin(async { Ok(None) }));
4125
4126 mocks
4128 .provider
4129 .expect_get_transaction_count()
4130 .returning(|_| Box::pin(async { Ok(10) }));
4131
4132 let tx_clone = tx.clone();
4134 mocks
4135 .tx_repo
4136 .expect_partial_update()
4137 .withf(|_, update| {
4138 update.status == Some(TransactionStatus::Failed)
4139 && update
4140 .status_reason
4141 .as_ref()
4142 .map(|r| r.contains("consumed externally"))
4143 .unwrap_or(false)
4144 })
4145 .returning(move |_, update| {
4146 let mut updated_tx = tx_clone.clone();
4147 updated_tx.status = update.status.unwrap();
4148 updated_tx.status_reason = update.status_reason.clone();
4149 Ok(updated_tx)
4150 });
4151
4152 mocks
4154 .job_producer
4155 .expect_produce_relayer_health_check_job()
4156 .withf(|job, scheduled_on| {
4157 job.relayer_id == "test-relayer-id"
4158 && job.metadata.as_ref().map_or(false, |m| {
4159 m.get("health_check_action") == Some(&"nonce_health".to_string())
4160 })
4161 && scheduled_on.is_none()
4162 })
4163 .returning(|_, _| Box::pin(async { Ok(()) }));
4164
4165 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4166 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
4167
4168 assert!(result.is_ok());
4169 let recovered = result.unwrap();
4170 assert!(recovered.is_some(), "Expected Some(tx) for consumed nonce");
4171 assert_eq!(recovered.unwrap().status, TransactionStatus::Failed);
4172 }
4173
4174 #[tokio::test]
4176 async fn test_nonce_recovery_nonce_not_consumed() {
4177 let mut mocks = default_test_mocks();
4178 let relayer = create_test_relayer();
4179
4180 let mut tx = make_test_transaction(TransactionStatus::Submitted);
4181 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4182 nonce: Some(5),
4183 hash: Some("0xhash".to_string()),
4184 raw: Some(vec![1, 2, 3]),
4185 ..tx.network_data.get_evm_transaction_data().unwrap()
4186 });
4187
4188 mocks
4190 .provider
4191 .expect_get_transaction_receipt()
4192 .returning(|_| Box::pin(async { Ok(None) }));
4193
4194 mocks
4196 .provider
4197 .expect_get_transaction_count()
4198 .returning(|_| Box::pin(async { Ok(5) }));
4199
4200 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4201 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
4202
4203 assert!(result.is_ok());
4204 assert!(
4205 result.unwrap().is_none(),
4206 "Expected None when nonce not consumed"
4207 );
4208 }
4209
4210 #[tokio::test]
4212 async fn test_nonce_recovery_receipt_found() {
4213 let mut mocks = default_test_mocks();
4214 let relayer = create_test_relayer();
4215
4216 let mut tx = make_test_transaction(TransactionStatus::Submitted);
4217 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4218 nonce: Some(5),
4219 hash: Some("0xhash".to_string()),
4220 raw: Some(vec![1, 2, 3]),
4221 ..tx.network_data.get_evm_transaction_data().unwrap()
4222 });
4223
4224 mocks
4226 .provider
4227 .expect_get_transaction_receipt()
4228 .returning(|_| {
4229 let receipt = make_mock_receipt(true, Some(100));
4230 Box::pin(async move { Ok(Some(receipt)) })
4231 });
4232
4233 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4234 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
4235
4236 assert!(result.is_ok());
4237 assert!(
4238 result.unwrap().is_none(),
4239 "Expected None when receipt found — defer to normal flow"
4240 );
4241 }
4242
4243 #[tokio::test]
4246 async fn test_nonce_recovery_rpc_error_prevents_force_fail() {
4247 let mut mocks = default_test_mocks();
4248 let relayer = create_test_relayer();
4249
4250 let mut tx = make_test_transaction(TransactionStatus::Submitted);
4251 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4252 nonce: Some(5),
4253 hash: Some("0xhash".to_string()),
4254 raw: Some(vec![1, 2, 3]),
4255 ..tx.network_data.get_evm_transaction_data().unwrap()
4256 });
4257
4258 mocks
4260 .provider
4261 .expect_get_transaction_receipt()
4262 .returning(|_| {
4263 Box::pin(async {
4264 Err(crate::services::provider::ProviderError::Other(
4265 "RPC timeout".to_string(),
4266 ))
4267 })
4268 });
4269
4270 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4274 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
4275
4276 assert!(result.is_ok());
4277 assert!(
4278 result.unwrap().is_none(),
4279 "Expected None when RPC errors occurred — must not force-fail on incomplete data"
4280 );
4281 }
4282
4283 #[tokio::test]
4285 async fn test_handle_status_impl_nonce_recovery_hint() {
4286 let mut mocks = default_test_mocks();
4287 let relayer = create_test_relayer();
4288
4289 let mut tx = make_test_transaction(TransactionStatus::Submitted);
4290 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4291 nonce: Some(5),
4292 hash: Some("0xhash".to_string()),
4293 raw: Some(vec![1, 2, 3]),
4294 ..tx.network_data.get_evm_transaction_data().unwrap()
4295 });
4296 tx.sent_at = Some(Utc::now().to_rfc3339());
4297
4298 mocks
4300 .provider
4301 .expect_get_transaction_receipt()
4302 .returning(|_| Box::pin(async { Ok(None) }));
4303
4304 mocks
4306 .provider
4307 .expect_get_transaction_count()
4308 .returning(|_| Box::pin(async { Ok(10) }));
4309
4310 let tx_clone = tx.clone();
4312 mocks
4313 .tx_repo
4314 .expect_partial_update()
4315 .returning(move |_, update| {
4316 let mut updated_tx = tx_clone.clone();
4317 if let Some(status) = update.status {
4318 updated_tx.status = status;
4319 }
4320 updated_tx.status_reason = update.status_reason.clone();
4321 Ok(updated_tx)
4322 });
4323
4324 mocks
4326 .job_producer
4327 .expect_produce_relayer_health_check_job()
4328 .returning(|_, _| Box::pin(async { Ok(()) }));
4329
4330 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4331
4332 let mut metadata = std::collections::HashMap::new();
4334 metadata.insert(
4335 TX_NONCE_RECONCILE_TRIGGER.to_string(),
4336 "NonceTooLow".to_string(),
4337 );
4338 let context = StatusCheckContext::default().with_job_metadata(Some(metadata));
4339
4340 let result = evm_transaction.handle_status_impl(tx, Some(context)).await;
4341 assert!(result.is_ok());
4342 assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4343 }
4344 }
4345
4346 mod circuit_breaker_sent_state_tests {
4347 use super::*;
4348 use crate::jobs::StatusCheckContext;
4349
4350 #[tokio::test]
4352 async fn test_circuit_breaker_sent_with_nonce_issues_noop() {
4353 let mut mocks = default_test_mocks_with_network();
4354 let relayer = create_test_relayer();
4355
4356 let mut tx = make_test_transaction(TransactionStatus::Sent);
4357 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4358 nonce: Some(5),
4359 hash: Some("0xhash".to_string()),
4360 raw: Some(vec![1, 2, 3]),
4361 ..tx.network_data.get_evm_transaction_data().unwrap()
4362 });
4363 tx.sent_at = Some(Utc::now().to_rfc3339());
4364
4365 let tx_clone = tx.clone();
4367 mocks
4368 .tx_repo
4369 .expect_partial_update()
4370 .returning(move |_, update| {
4371 let mut updated_tx = tx_clone.clone();
4372 if let Some(status) = update.status {
4373 updated_tx.status = status;
4374 }
4375 updated_tx.status_reason = update.status_reason.clone();
4376 Ok(updated_tx)
4377 });
4378
4379 mocks
4381 .job_producer
4382 .expect_produce_submit_transaction_job()
4383 .times(1)
4384 .returning(|_, _| Box::pin(async { Ok(()) }));
4385
4386 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4387
4388 let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4390
4391 let result = evm_transaction
4392 .handle_circuit_breaker_safely(tx, &ctx)
4393 .await;
4394 assert!(result.is_ok());
4395 }
4396
4397 #[tokio::test]
4399 async fn test_circuit_breaker_sent_without_nonce_marks_failed() {
4400 let mut mocks = default_test_mocks();
4401 let relayer = create_test_relayer();
4402
4403 let mut tx = make_test_transaction(TransactionStatus::Sent);
4404 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4406 nonce: None,
4407 hash: None,
4408 raw: None,
4409 ..tx.network_data.get_evm_transaction_data().unwrap()
4410 });
4411 tx.sent_at = Some(Utc::now().to_rfc3339());
4412
4413 let tx_clone = tx.clone();
4415 mocks
4416 .tx_repo
4417 .expect_partial_update()
4418 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
4419 .returning(move |_, update| {
4420 let mut updated_tx = tx_clone.clone();
4421 updated_tx.status = update.status.unwrap();
4422 Ok(updated_tx)
4423 });
4424
4425 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4426
4427 let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4428
4429 let result = evm_transaction
4430 .handle_circuit_breaker_safely(tx, &ctx)
4431 .await;
4432 assert!(result.is_ok());
4433 assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4434 }
4435
4436 #[tokio::test]
4438 async fn test_circuit_breaker_pending_marks_failed() {
4439 let mut mocks = default_test_mocks();
4440 let relayer = create_test_relayer();
4441
4442 let tx = make_test_transaction(TransactionStatus::Pending);
4443
4444 let tx_clone = tx.clone();
4446 mocks
4447 .tx_repo
4448 .expect_partial_update()
4449 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
4450 .returning(move |_, update| {
4451 let mut updated_tx = tx_clone.clone();
4452 updated_tx.status = update.status.unwrap();
4453 Ok(updated_tx)
4454 });
4455
4456 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4457
4458 let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4459
4460 let result = evm_transaction
4461 .handle_circuit_breaker_safely(tx, &ctx)
4462 .await;
4463 assert!(result.is_ok());
4464 assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4465 }
4466 }
4467}