chore(coprocessor): update config for gw-listener and txn-sender

Update default values in both Rust code and the charts. Namely, make
retries for stuff that is supposed to eventually succeed to practically
infinite.

Also, add all missing parameters in the chart file for gw-listener and
txn-sender.

In general, update default values to more sensible ones for prod.

Author: dartdart26

charts/coprocessor/Chart.yaml

@@ -1,6 +1,6 @@
 name: coprocessor
 description: A helm chart to distribute and deploy Zama fhevm Co-Processor services
-version: 0.6.2
+version: 0.7.0
 apiVersion: v2
 keywords:
   - fhevm

charts/coprocessor/values.yaml

@@ -234,16 +234,21 @@ gwListener:
   # Command line arguments for the gateway listener
   args:
     - --database-url=$(DATABASE_URL)
+    - --database-pool-size=16
+    - --verify-proof-req-database-channel=event_zkpok_new_work
     - --gw-url=ws://gateway-rpc-node:8548
     - --input-verification-address=$(INPUT_VERIFICATION_ADDRESS)
-    - --verify-proof-req-database-channel="event_zkpok_new_work"
-    - --database-pool-size=16
+    - --kms-generation-address=$(KMS_GENERATION_ADDRESS)
     - --error-sleep-initial-secs=1
     - --error-sleep-max-secs=10
+    - --health-check-port=8080
+    - --metrics-addr=0.0.0.0:9100
+    - --provider-max-retries=4294967295
+    - --provider-retry-interval=4s
+    - --log-level=INFO
     - --get-logs-poll-interval=1s
     - --get-logs-block-batch-size=100
-    # NEW ARG: KMS generation contract address
-    - --kms-generation-address=$(KMS_GENERATION_ADDRESS)
+    - --service-name=gw-listener
     ### Catchup parameters (optional)
     # --catchup-kms-generation-from-block BLOCK_NUMBER
     # To go back in time from latest block
@@ -456,8 +461,8 @@ zkProofWorker:
   # Command line arguments for ZK proof worker
   args:
     - --database-url=$(DATABASE_URL)
-    - --pg-listen-channel="event_zkpok_new_work"
-    - --pg-notify-channel="event_zkpok_computed"
+    - --pg-listen-channel=event_zkpok_new_work
+    - --pg-notify-channel=event_zkpok_computed
     - --pg-polling-interval=5
     - --pg-pool-connections=5
     - --pg-timeout=15s
@@ -694,24 +699,59 @@ txSender:
           key: coprocessor-db-url
     - name: TX_SENDER_PRIVATE_KEY
       value: "0x8f82b3f482c19a95ac29c82cf048c076ed0de2530c64a73f2d2d7d1e64b5cc6e"
-    - name: ZKPOK_MANAGER_ADDRESS
+    - name: INPUT_VERIFICATION_ADDRESS
       valueFrom:
         configMapKeyRef:
           name: gateway-sc-addresses
-          key: zkpok_manager.address
-    - name: CIPHERTEXT_STORAGE_ADDRESS
+          key: input_verification.address
+    - name: CIPHERTEXT_COMMITS_ADDRESS
       valueFrom:
         configMapKeyRef:
           name: gateway-sc-addresses
-          key: fhevm.address
+          key: ciphertext_commits.address
+    - name: MULTICHAIN_ACL_ADDRESS
+      valueFrom:
+        configMapKeyRef:
+          name: gw-sc-addresses
+          key: multichain_acl.address
 
   # Command line arguments for transaction sender
   args:
-    - --database-url=$(DATABASE_URL)
-    - --gateway-url=http://gateway-anvil-node:8546
+    - --input-verification-address=$(INPUT_VERIFICATION_ADDRESS)
+    - --ciphertext-commits-address=$(CIPHERTEXT_COMMITS_ADDRESS)
+    - --multichain-acl-address=$(MULTICHAIN_ACL_ADDRESS)
+    - --gateway-url=ws://gateway-rpc-node:8548
+    - --signer-type=private-key
     - --private-key=$(TX_SENDER_PRIVATE_KEY)
-    - --zkpok-manager-address=$(ZKPOK_MANAGER_ADDRESS)
-    - --ciphertext-storage-address=$(CIPHERTEXT_STORAGE_ADDRESS)
+    - --database-url=$(DATABASE_URL)
+    - --database-pool-size=10
+    - --database-polling-interval-secs=1
+    - --verify-proof-resp-database-channel=event_zkpok_computed
+    - --add-ciphertexts-database-channel=event_ciphertexts_uploaded
+    - --allow-handle-database-channel=event_allowed_handle
+    - --verify-proof-resp-batch-limit=128
+    - --verify-proof-resp-max-retries=6
+    - --verify-proof-remove-after-max-retries
+    - --add-ciphertexts-batch-limit=10
+    - --allow-handle-batch-limit=10
+    - --allow-handle-max-retries=2147483647
+    - --add-ciphertexts-max-retries=2147483647
+    - --error-sleep-initial-secs=1
+    - --error-sleep-max-secs=300
+    - --txn-receipt-timeout-secs=10
+    - --required-txn-confirmations=0
+    - --review-after-unlimited-retries=30
+    - --provider-max-retries=4294967295
+    - --provider-retry-interval=4s
+    - --health-check-port=8080
+    - --metrics-addr=0.0.0.0:9100
+    - --health-check-timeout=4s
+    - --log-level=INFO
+    - --gas-limit-overprovision-percent=120
+    - --graceful-shutdown-timeout=8s
+    - --service-name=txn-sender
+    - --metric-host-txn-latency=0.1:60.0:0.1
+    - --metric-zkproof-txn-latency=0.1:60.0:0.1
 
   # Service ports configuration
   ports:

coprocessor/README.md

@@ -145,81 +145,47 @@ Options:
 
 ```bash
 $ gw_listener --help
-Usage: gw_listener [OPTIONS] --gw-url <GW_URL> --input-verification-address <INPUT_VERIFICATION_ADDRESS> --kms-management-address <KMS_MANAGEMENT_ADDRESS>
+Usage: gw_listener [OPTIONS] --gw-url <GW_URL> --input-verification-address <INPUT_VERIFICATION_ADDRESS> --kms-generation-address <KMS_GENERATION_ADDRESS>
 
 Options:
       --database-url <DATABASE_URL>
+          
       --database-pool-size <DATABASE_POOL_SIZE>
           [default: 16]
       --verify-proof-req-database-channel <VERIFY_PROOF_REQ_DATABASE_CHANNEL>
-          [default: verify_proof_requests]
+          [default: event_zkpok_new_work]
       --gw-url <GW_URL>
+          
   -i, --input-verification-address <INPUT_VERIFICATION_ADDRESS>
-      --kms-management-address <KMS_MANAGEMENT_ADDRESS>
+          
+      --kms-generation-address <KMS_GENERATION_ADDRESS>
+          
       --error-sleep-initial-secs <ERROR_SLEEP_INITIAL_SECS>
           [default: 1]
       --error-sleep-max-secs <ERROR_SLEEP_MAX_SECS>
           [default: 10]
-      --catchup-kms-generation-from-block <BLOCK_NUMBER OR -BLOCKS_BACK>
-          [default: None]
-  -h, --help
-          Print help
-  -V, --version
-          Print version
-```
-
-For more info, please check [gw-listener README](fhevm-engine/gw-listener/README.md)
-
-##### sns-worker
-
-```bash
-$ sns_worker --help
-Usage: sns_worker [OPTIONS] --pg-listen-channel <PG_LISTEN_CHANNEL> --pg-notify-channel <PG_NOTIFY_CHANNEL>
-
-Options:
-      --work-items-batch-size <WORK_ITEMS_BATCH_SIZE>
-          Work items batch size [default: 4]
-      --pg-listen-channel <PG_LISTEN_CHANNEL>
-          NOTIFY/LISTEN channel for database that the worker listen to
-      --pg-notify-channel <PG_NOTIFY_CHANNEL>
-          NOTIFY/LISTEN channel for database that the worker notify to
-      --pg-polling-interval <PG_POLLING_INTERVAL>
-          Polling interval in seconds [default: 60]
-      --pg-pool-connections <PG_POOL_CONNECTIONS>
-          Postgres pool connections [default: 10]
-      --database-url <DATABASE_URL>
-          Postgres database url. If unspecified DATABASE_URL environment variable is used
-      --keys-file-path <KEYS_FILE_PATH>
-          KeySet file. If unspecified the the keys are read from the database (not implemented)
+      --health-check-port <HEALTH_CHECK_PORT>
+          [default: 8080]
+      --metrics-addr <METRICS_ADDR>
+          Prometheus metrics server address [default: 0.0.0.0:9100]
+      --health-check-timeout <HEALTH_CHECK_TIMEOUT>
+          [default: 4s]
+      --provider-max-retries <PROVIDER_MAX_RETRIES>
+          [default: 4294967295]
+      --provider-retry-interval <PROVIDER_RETRY_INTERVAL>
+          [default: 4s]
+      --log-level <LOG_LEVEL>
+          [default: INFO]
+      --host-chain-id <HOST_CHAIN_ID>
+          
+      --get-logs-poll-interval <GET_LOGS_POLL_INTERVAL>
+          [default: 1s]
+      --get-logs-block-batch-size <GET_LOGS_BLOCK_BATCH_SIZE>
+          [default: 100]
       --service-name <SERVICE_NAME>
-          sns-executor service name in OTLP traces (not implemented) [default: sns-executor]
-  -h, --help
-          Print help
-  -V, --version
-          Print version
-```
-
-##### zkproof-worker
-
-```bash
-$ zkproof_worker --help
-Usage: zkproof_worker [OPTIONS]
-
-Options:
-  -d, --database-url <DATABASE_URL>
-
-      --database-pool-size <DATABASE_POOL_SIZE>
-          [default: 10]
-      --database-polling-interval-secs <DATABASE_POLLING_INTERVAL_SECS>
-          [default: 5]
-  -v, --verify-proof-req-database-channel <VERIFY_PROOF_REQ_DATABASE_CHANNEL>
-          [default: verify_proof_resquests]
-  -t, --tokio-blocking-threads <TOKIO_BLOCKING_THREADS>
-          [default: 16]
-      --error-sleep-initial-secs <ERROR_SLEEP_INITIAL_SECS>
-          [default: 1]
-      --error-sleep-max-secs <ERROR_SLEEP_MAX_SECS>
-          [default: 10]
+          gw-listener service name in OTLP traces [default: gw-listener]
+      --catchup-kms-generation-from-block <CATCHUP_KMS_GENERATION_FROM_BLOCK>
+          Can be negative from last processed block
   -h, --help
           Print help
   -V, --version
@@ -250,37 +216,59 @@ Options:
       --database-pool-size <DATABASE_POOL_SIZE>
           [default: 10]
       --database-polling-interval-secs <DATABASE_POLLING_INTERVAL_SECS>
-          [default: 5]
+          [default: 1]
       --verify-proof-resp-database-channel <VERIFY_PROOF_RESP_DATABASE_CHANNEL>
-          [default: verify_proof_responses]
+          [default: event_zkpok_computed]
       --add-ciphertexts-database-channel <ADD_CIPHERTEXTS_DATABASE_CHANNEL>
-          [default: add_ciphertexts]
+          [default: event_ciphertexts_uploaded]
       --allow-handle-database-channel <ALLOW_HANDLE_DATABASE_CHANNEL>
           [default: event_allowed_handle]
       --verify-proof-resp-batch-limit <VERIFY_PROOF_RESP_BATCH_LIMIT>
           [default: 128]
       --verify-proof-resp-max-retries <VERIFY_PROOF_RESP_MAX_RETRIES>
-          [default: 3]
+          [default: 6]
       --verify-proof-remove-after-max-retries
 
       --add-ciphertexts-batch-limit <ADD_CIPHERTEXTS_BATCH_LIMIT>
           [default: 10]
       --allow-handle-batch-limit <ALLOW_HANDLE_BATCH_LIMIT>
           [default: 10]
       --allow-handle-max-retries <ALLOW_HANDLE_MAX_RETRIES>
-          [default: 10]
+          [default: 4294967295]
       --add-ciphertexts-max-retries <ADD_CIPHERTEXTS_MAX_RETRIES>
-          [default: 15]
+          [default: 4294967295]
       --error-sleep-initial-secs <ERROR_SLEEP_INITIAL_SECS>
           [default: 1]
       --error-sleep-max-secs <ERROR_SLEEP_MAX_SECS>
-          [default: 16]
+          [default: 300]
       --txn-receipt-timeout-secs <TXN_RECEIPT_TIMEOUT_SECS>
           [default: 10]
       --required-txn-confirmations <REQUIRED_TXN_CONFIRMATIONS>
           [default: 0]
       --review-after-unlimited-retries <REVIEW_AFTER_UNLIMITED_RETRIES>
           [default: 30]
+      --provider-max-retries <PROVIDER_MAX_RETRIES>
+          [default: 4294967295]
+      --provider-retry-interval <PROVIDER_RETRY_INTERVAL>
+          [default: 4s]
+      --health-check-port <HEALTH_CHECK_PORT>
+          [default: 8080]
+      --metrics-addr <METRICS_ADDR>
+          Prometheus metrics server address [default: 0.0.0.0:9100]
+      --health-check-timeout <HEALTH_CHECK_TIMEOUT>
+          [default: 4s]
+      --log-level <LOG_LEVEL>
+          [default: INFO]
+      --gas-limit-overprovision-percent <GAS_LIMIT_OVERPROVISION_PERCENT>
+          [default: 120]
+      --graceful-shutdown-timeout <GRACEFUL_SHUTDOWN_TIMEOUT>
+          [default: 8s]
+      --service-name <SERVICE_NAME>
+          service name in OTLP traces [default: txn-sender]
+      --metric-host-txn-latency <METRIC_HOST_TXN_LATENCY>
+          Prometheus metrics: coprocessor_host_txn_latency_seconds [default: 0.1:60.0:0.1]
+      --metric-zkproof-txn-latency <METRIC_ZKPROOF_TXN_LATENCY>
+          Prometheus metrics: coprocessor_zkproof_txn_latency_seconds [default: 0.1:60.0:0.1]
   -h, --help
           Print help
   -V, --version

coprocessor/fhevm-engine/gw-listener/README.md

@@ -6,7 +6,7 @@ The **gw-listener** service listens for events from the GW and dispatches them t
 
 **gw-listener** listens for input proof verification events from the InputVerification contract and inserts them into the DB into the `verify_proofs` table.
 
-The gw-listener will notify **zkproof-worker** services that work is available over the `verify_proof_requests` DB channel (configurable, but this is the default one).
+The gw-listener will notify **zkproof-worker** services that work is available over the `event_zkpok_new_work` DB channel (configurable, but this is the default one).
 
 Once a ZK proof request is verified, a zkproof-worker should set:
  * `verified = true or false`

coprocessor/fhevm-engine/gw-listener/src/bin/gw_listener.rs

@@ -20,10 +20,10 @@ struct Conf {
     #[arg(long)]
     database_url: Option<String>,
 
-    #[arg(long, default_value = "16")]
+    #[arg(long, default_value_t = 16)]
     database_pool_size: u32,
 
-    #[arg(long, default_value = "verify_proof_requests")]
+    #[arg(long, default_value = "event_zkpok_new_work")]
     verify_proof_req_database_channel: String,
 
     #[arg(long)]
@@ -35,10 +35,10 @@ struct Conf {
     #[arg(long)]
     kms_generation_address: Address,
 
-    #[arg(long, default_value = "1")]
+    #[arg(long, default_value_t = 1)]
     error_sleep_initial_secs: u16,
 
-    #[arg(long, default_value = "10")]
+    #[arg(long, default_value_t = 10)]
     error_sleep_max_secs: u16,
 
     #[arg(long, default_value_t = 8080)]
@@ -51,7 +51,7 @@ struct Conf {
     #[arg(long, default_value = "4s", value_parser = parse_duration)]
     health_check_timeout: Duration,
 
-    #[arg(long, default_value = "1000000")]
+    #[arg(long, default_value_t = u32::MAX)]
     provider_max_retries: u32,
 
     #[arg(long, default_value = "4s", value_parser = parse_duration)]

coprocessor/fhevm-engine/gw-listener/src/lib.rs

@@ -70,7 +70,7 @@ impl Default for ConfigSettings {
             database_url: std::env::var("DATABASE_URL")
                 .unwrap_or("postgres://postgres:postgres@localhost/coprocessor".to_owned()),
             database_pool_size: 16,
-            verify_proof_req_db_channel: "verify_proof_requests".to_owned(),
+            verify_proof_req_db_channel: "event_zkpok_new_work".to_owned(),
             gw_url: "ws://127.0.0.1:8546".try_into().expect("Invalid URL"),
             error_sleep_initial_secs: 1,
             error_sleep_max_secs: 10,