diff --git a/tests/e2e_operations.py b/tests/e2e_operations.py index d6193e8f8d5..a73ec90ad89 100644 --- a/tests/e2e_operations.py +++ b/tests/e2e_operations.py @@ -1428,6 +1428,29 @@ def run_initial_tcb_version_checks(const_args): assert False, "No TCB_version found in recovery ledger" +def wait_for_sealed_secrets(node, min_seqno=0, timeout=10): + out, _ = node.remote.get_logs() + start = time.time() + while time.time() < start + timeout: + with open(out, "r") as outf: + for line in outf.readlines(): + if "Sealing complete of ledger secret to" in line: + try: + path = line.split()[-1] + filename = os.path.basename(path) + seqno = int(filename.split(".")[0]) + if seqno >= min_seqno: + return + except (IndexError, ValueError): + continue + + time.sleep(0.1) + + raise TimeoutError( + f"Could not find sealed secrets for seqno {min_seqno} after {timeout}s in logs" + ) + + def run_recovery_local_unsealing( const_args, recovery_f=0, rekey=False, recovery_shares_refresh=False ): @@ -1444,10 +1467,19 @@ def run_recovery_local_unsealing( primary, _ = network.find_primary() if rekey: + network.wait_for_node_commit_sync() + with primary.client() as c: + r = c.get("/node/commit").body.json() + min_seqno = TxID.from_str(r["transaction_id"]).seqno network.consortium.trigger_ledger_rekey(primary) + else: + min_seqno = 0 if recovery_shares_refresh: network.consortium.trigger_recovery_shares_refresh(primary) + for node in network.nodes: + wait_for_sealed_secrets(node, min_seqno=min_seqno) + node_secret_map = { node.local_node_id: node.save_sealed_ledger_secret() for node in network.nodes @@ -1503,6 +1535,8 @@ def run_recovery_unsealing_validate_audit(const_args): network.start_and_open(args) network.save_service_identity(args) + for node in network.nodes: + wait_for_sealed_secrets(node) node0_secrets = network.nodes[0].save_sealed_ledger_secret() latest_public_tables, _ = network.get_latest_ledger_public_state() @@ -1586,6 +1620,8 @@ def run_recovery_unsealing_corrupt(const_args, recovery_f=0): network.start_and_open(args) network.save_service_identity(args) + for node in network.nodes: + wait_for_sealed_secrets(node) node_secret_map = { node.local_node_id: node.save_sealed_ledger_secret()