Skip to content

Commit edfb00f

Browse files
authored
Increase checkpoint message size limit in ingestion client (#24313) (#24329)
## Description We were seeing client-side errors due to messages that exceeded the default 4mb limit. Bumping it up to 128mb (the biggest I've seen so far is ~90mb).
1 parent f8988d9 commit edfb00f

File tree

3 files changed

+10
-2
lines changed

3 files changed

+10
-2
lines changed

crates/sui-indexer-alt-framework/src/ingestion/client.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ use tracing::{debug, error, warn};
1616
use url::Url;
1717

1818
use crate::ingestion::Error as IngestionError;
19+
use crate::ingestion::MAX_GRPC_MESSAGE_SIZE_BYTES;
1920
use crate::ingestion::Result as IngestionResult;
2021
use crate::ingestion::local_client::LocalIngestionClient;
2122
use crate::ingestion::remote_client::RemoteIngestionClient;
@@ -105,9 +106,12 @@ impl IngestionClient {
105106
let client = if let Some(username) = username {
106107
let mut headers = HeadersInterceptor::new();
107108
headers.basic_auth(username, password);
108-
Client::new(url.to_string())?.with_headers(headers)
109+
Client::new(url.to_string())?
110+
.with_headers(headers)
111+
.with_max_decoding_message_size(MAX_GRPC_MESSAGE_SIZE_BYTES)
109112
} else {
110113
Client::new(url.to_string())?
114+
.with_max_decoding_message_size(MAX_GRPC_MESSAGE_SIZE_BYTES)
111115
};
112116
Ok(Self::new_impl(Arc::new(client), metrics))
113117
}

crates/sui-indexer-alt-framework/src/ingestion/mod.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ mod streaming_client;
3131
#[cfg(test)]
3232
mod test_utils;
3333

34+
pub(crate) const MAX_GRPC_MESSAGE_SIZE_BYTES: usize = 128 * 1024 * 1024;
35+
3436
#[derive(clap::Args, Clone, Debug, Default)]
3537
#[group(required = true)]
3638
pub struct ClientArgs {

crates/sui-indexer-alt-framework/src/ingestion/streaming_client.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ use sui_rpc::proto::sui::rpc::v2::{
1111
use sui_rpc_api::client::checkpoint_data_field_mask;
1212
use tonic::{Status, transport::Uri};
1313

14+
use crate::ingestion::MAX_GRPC_MESSAGE_SIZE_BYTES;
1415
use crate::ingestion::error::{Error, Result};
1516
use crate::types::full_checkpoint_content::Checkpoint;
1617

@@ -39,7 +40,8 @@ impl CheckpointStreamingClient for GrpcStreamingClient {
3940
async fn connect(&mut self) -> Result<CheckpointStream> {
4041
let mut client = SubscriptionServiceClient::connect(self.uri.clone())
4142
.await
42-
.map_err(|err| Error::RpcClientError(Status::from_error(err.into())))?;
43+
.map_err(|err| Error::RpcClientError(Status::from_error(err.into())))?
44+
.max_decoding_message_size(MAX_GRPC_MESSAGE_SIZE_BYTES);
4345

4446
let mut request = SubscribeCheckpointsRequest::default();
4547
request.read_mask = Some(checkpoint_data_field_mask());

0 commit comments

Comments
 (0)