Wait for sealed secrets

tests/e2e_operations.py

@@ -1428,6 +1428,30 @@ def run_initial_tcb_version_checks(const_args):
             assert False, "No TCB_version found in recovery ledger"
 
 
+def wait_for_sealed_secrets(node, min_seqno=0, timeout=10):
+    out, _ = node.remote.get_logs()
+    start = time.time()
+    while True:
+        with open(out, "r") as outf:
+            for line in outf.readlines():
+                # LOG_INFO_FMT("Sealing complete of ledger secret to {}", sealing_path);
+                if "Sealing complete of ledger secret to" in line:
+                    try:
+                        path = line.split()[-1]
+                        filename = os.path.basename(path)
+                        seqno = int(filename.split(".")[0])
+                        if seqno >= min_seqno:
+                            return
+                    except (IndexError, ValueError):
+                        continue
+
+        if time.time() > start + timeout:
+            raise TimeoutError(
+                f"Could not find sealed secrets for seqno {min_seqno} after {timeout}s in logs"
+            )
+        time.sleep(0.1)
+
+
 def run_recovery_local_unsealing(
     const_args, recovery_f=0, rekey=False, recovery_shares_refresh=False
 ):
@@ -1444,10 +1468,19 @@ def run_recovery_local_unsealing(
 
         primary, _ = network.find_primary()
         if rekey:
+            time.sleep(1)  # Ensure that the network is stable before proceeding
+            with primary.client() as c:
+                r = c.get("/node/commit").body.json()
+                min_seqno = TxID.from_str(r["transaction_id"]).seqno
             network.consortium.trigger_ledger_rekey(primary)
+        else:
+            min_seqno = 0
+
         if recovery_shares_refresh:
             network.consortium.trigger_recovery_shares_refresh(primary)
 
+        wait_for_sealed_secrets(primary, min_seqno=min_seqno)
+
         node_secret_map = {
             node.local_node_id: node.save_sealed_ledger_secret()
             for node in network.nodes
@@ -1503,6 +1536,7 @@ def run_recovery_unsealing_validate_audit(const_args):
         network.start_and_open(args)
 
         network.save_service_identity(args)
+        wait_for_sealed_secrets(network.nodes[0])
         node0_secrets = network.nodes[0].save_sealed_ledger_secret()
 
         latest_public_tables, _ = network.get_latest_ledger_public_state()
@@ -1586,6 +1620,7 @@ def run_recovery_unsealing_corrupt(const_args, recovery_f=0):
         network.start_and_open(args)
 
         network.save_service_identity(args)
+        wait_for_sealed_secrets(network.nodes[0])
 
         node_secret_map = {
             node.local_node_id: node.save_sealed_ledger_secret()