[MFP] enhance metrics with additional labels. Refactor tx retry back off (#23725)

## Description 

This PR:

* Enhances some metrics with additional labels
* Refactors the back off strategy when retrying the tx submission

Running for `10` times the back off with the current settings yields
results like:

```
58.618524ms
1.717898302s
626.285083ms
8.890868961s
6.853596566s
1.27459605s
8.60306108s
2.898520074s
1.538756239s
2.389480396s
```

Although we are using an exponential back off, the current setup and
jitter seems to mess up a bit the back off latency to be used. It's not
very scalar in the sense that we should expect using a bigger and bigger
step every time.

With the proposed changes we'll have something like:

```
136.153155ms
172.376361ms
315.614501ms
460.543044ms
866.642052ms
1.196423s
1.584154394s
3.647710108s
4.406361196s
8.318059196s
```

## Test plan 

How did you test the new or updated feature?

---

## Release notes

Check each box that your changes affect. If none of the boxes relate to
your changes, release notes aren't required.

For each box you select, include information after the relevant heading
that describes the impact of your changes that a user might notice and
any actions they must take to implement updates.

- [ ] Protocol: 
- [ ] Nodes (Validators and Full nodes): 
- [ ] gRPC:
- [ ] JSON-RPC: 
- [ ] GraphQL: 
- [ ] CLI: 
- [ ] Rust SDK:

Author: akichidis

crates/sui-core/src/transaction_driver/metrics.rs

@@ -72,14 +72,14 @@ impl TransactionDriverMetrics {
             validator_submit_transaction_errors: register_int_counter_vec_with_registry!(
                 "transaction_driver_validator_submit_transaction_errors",
                 "Number of submit transaction errors by validator",
-                &["validator", "error_type"],
+                &["validator", "error_type", "tx_type", "ping"],
                 registry,
             )
             .unwrap(),
             validator_submit_transaction_successes: register_int_counter_vec_with_registry!(
                 "transaction_driver_validator_submit_transaction_successes",
                 "Number of successful submit transactions by validator",
-                &["validator"],
+                &["validator", "tx_type", "ping"],
                 registry,
             )
             .unwrap(),
@@ -149,7 +149,7 @@ impl TransactionDriverMetrics {
             validator_selections: register_int_counter_vec_with_registry!(
                 "transaction_driver_validator_selections",
                 "Number of times each validator was selected for transaction submission",
-                &["validator"],
+                &["validator", "tx_type", "ping"],
                 registry,
             )
             .unwrap(),

crates/sui-core/src/transaction_driver/mod.rs

@@ -10,7 +10,7 @@ mod transaction_submitter;
 /// Exports
 pub use error::TransactionDriverError;
 pub use metrics::*;
-use tokio_retry::strategy::{jitter, ExponentialBackoff};
+use tokio_retry::strategy::ExponentialBackoff;
 
 use std::{
     net::SocketAddr,
@@ -263,7 +263,7 @@ where
         // Exponential backoff with jitter to prevent thundering herd on retries
         let mut backoff = ExponentialBackoff::from_millis(100)
             .max_delay(MAX_RETRY_DELAY)
-            .map(jitter);
+            .map(|duration| duration.mul_f64(rand::thread_rng().gen_range(0.5..1.0)));
         let mut attempts = 0;
         let mut latest_retriable_error = None;
 

crates/sui-core/src/transaction_driver/transaction_submitter.rs

@@ -72,6 +72,12 @@ impl TransactionSubmitter {
             tx_type,
             options.allowed_validators.clone(),
         );
+
+        let ping_label = if request.ping.is_some() {
+            "true"
+        } else {
+            "false"
+        };
         let mut retries = 0;
         let mut request_rpcs = FuturesUnordered::new();
 
@@ -85,7 +91,7 @@ impl TransactionSubmitter {
                         let display_name = authority_aggregator.get_display_name(&name);
                         self.metrics
                             .validator_selections
-                            .with_label_values(&[&display_name])
+                            .with_label_values(&[&display_name, tx_type.as_str(), ping_label])
                             .inc();
 
                         // Create a future that returns the name and display_name along with the result
@@ -132,7 +138,7 @@ impl TransactionSubmitter {
                 Some((name, display_name, Ok(result))) => {
                     self.metrics
                         .validator_submit_transaction_successes
-                        .with_label_values(&[&display_name])
+                        .with_label_values(&[&display_name, tx_type.as_str(), ping_label])
                         .inc();
                     self.metrics
                         .submit_transaction_retries
@@ -150,7 +156,12 @@ impl TransactionSubmitter {
                     };
                     self.metrics
                         .validator_submit_transaction_errors
-                        .with_label_values(&[&display_name, error_type])
+                        .with_label_values(&[
+                            &display_name,
+                            error_type,
+                            tx_type.as_str(),
+                            ping_label,
+                        ])
                         .inc();
 
                     retries += 1;