Skip to content

Commit c21ec5d

Browse files
swamirishiswamirishi
authored
HDDS-13847. Introduce Snapshot Content Lock to lock table contents (#9212)
1 parent 72167cf commit c21ec5d

File tree

13 files changed

+131
-23
lines changed

13 files changed

+131
-23
lines changed

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/FlatResource.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,10 @@ public enum FlatResource implements Resource {
2828
// Lock acquired on a Snapshot's RocksDB Handle.
2929
SNAPSHOT_DB_LOCK("SNAPSHOT_DB_LOCK"),
3030
// Lock acquired on a Snapshot's Local Data.
31-
SNAPSHOT_LOCAL_DATA_LOCK("SNAPSHOT_LOCAL_DATA_LOCK");
31+
SNAPSHOT_LOCAL_DATA_LOCK("SNAPSHOT_LOCAL_DATA_LOCK"),
32+
// Lock acquired on a Snapshot's RocksDB contents.
33+
SNAPSHOT_DB_CONTENT_LOCK("SNAPSHOT_DB_CONTENT_LOCK");
34+
3235

3336
private String name;
3437
private IOzoneManagerLock.ResourceManager resourceManager;

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotDefragService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
9393
snapshotsDefraggedCount = new AtomicLong(0);
9494
running = new AtomicBoolean(false);
9595
IOzoneManagerLock omLock = ozoneManager.getMetadataManager().getLock();
96-
this.snapshotIdLocks = new MultiSnapshotLocks(omLock, SNAPSHOT_GC_LOCK, true);
96+
this.snapshotIdLocks = new MultiSnapshotLocks(omLock, SNAPSHOT_GC_LOCK, true, 1);
9797
}
9898

9999
@Override

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@
2222
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE;
2323
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
2424
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_INFO_TABLE;
25+
import static org.apache.hadoop.ozone.om.lock.FlatResource.SNAPSHOT_DB_CONTENT_LOCK;
2526

2627
import com.google.common.annotations.VisibleForTesting;
2728
import jakarta.annotation.Nonnull;
2829
import java.io.IOException;
2930
import java.util.List;
3031
import java.util.Map;
32+
import java.util.UUID;
3133
import org.apache.commons.lang3.tuple.Pair;
3234
import org.apache.hadoop.hdds.utils.db.BatchOperation;
3335
import org.apache.hadoop.hdds.utils.db.DBStore;
@@ -36,11 +38,14 @@
3638
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
3739
import org.apache.hadoop.ozone.om.OmSnapshot;
3840
import org.apache.hadoop.ozone.om.OmSnapshotManager;
41+
import org.apache.hadoop.ozone.om.exceptions.OMException;
3942
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
4043
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
4144
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
4245
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
4346
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
47+
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
48+
import org.apache.hadoop.ozone.om.lock.OMLockDetails;
4449
import org.apache.hadoop.ozone.om.request.key.OMDirectoriesPurgeRequestWithFSO;
4550
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
4651
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -86,9 +91,15 @@ public void addToDBBatch(OMMetadataManager metadataManager,
8691
OmSnapshotManager omSnapshotManager =
8792
((OmMetadataManagerImpl) metadataManager)
8893
.getOzoneManager().getOmSnapshotManager();
89-
94+
IOzoneManagerLock lock = metadataManager.getLock();
95+
UUID fromSnapshotId = fromSnapshotInfo.getSnapshotId();
96+
OMLockDetails lockDetails = lock.acquireReadLock(SNAPSHOT_DB_CONTENT_LOCK, fromSnapshotId.toString());
97+
if (!lockDetails.isLockAcquired()) {
98+
throw new OMException("Unable to acquire read lock on " + SNAPSHOT_DB_CONTENT_LOCK + " for snapshot: " +
99+
fromSnapshotId, OMException.ResultCodes.INTERNAL_ERROR);
100+
}
90101
try (UncheckedAutoCloseableSupplier<OmSnapshot>
91-
rcFromSnapshotInfo = omSnapshotManager.getSnapshot(fromSnapshotInfo.getSnapshotId())) {
102+
rcFromSnapshotInfo = omSnapshotManager.getSnapshot(fromSnapshotId)) {
92103
OmSnapshot fromSnapshot = rcFromSnapshotInfo.get();
93104
DBStore fromSnapshotStore = fromSnapshot.getMetadataManager()
94105
.getStore();
@@ -98,6 +109,8 @@ public void addToDBBatch(OMMetadataManager metadataManager,
98109
processPaths(metadataManager, fromSnapshot.getMetadataManager(), batchOp, writeBatch);
99110
fromSnapshotStore.commitBatchOperation(writeBatch);
100111
}
112+
} finally {
113+
lock.releaseReadLock(SNAPSHOT_DB_CONTENT_LOCK, fromSnapshotId.toString());
101114
}
102115
metadataManager.getSnapshotInfoTable().putWithBatch(batchOp, fromSnapshotInfo.getTableKey(), fromSnapshotInfo);
103116
} else {

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,26 @@
1919

2020
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE;
2121
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_INFO_TABLE;
22+
import static org.apache.hadoop.ozone.om.lock.FlatResource.SNAPSHOT_DB_CONTENT_LOCK;
2223
import static org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotMoveDeletedKeysResponse.createRepeatedOmKeyInfo;
2324

2425
import jakarta.annotation.Nonnull;
2526
import java.io.IOException;
2627
import java.util.Collections;
2728
import java.util.List;
29+
import java.util.UUID;
2830
import org.apache.hadoop.hdds.utils.db.BatchOperation;
2931
import org.apache.hadoop.hdds.utils.db.DBStore;
3032
import org.apache.hadoop.ozone.om.OMMetadataManager;
3133
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
3234
import org.apache.hadoop.ozone.om.OmSnapshot;
3335
import org.apache.hadoop.ozone.om.OmSnapshotManager;
36+
import org.apache.hadoop.ozone.om.exceptions.OMException;
3437
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
3538
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
3639
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
40+
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
41+
import org.apache.hadoop.ozone.om.lock.OMLockDetails;
3742
import org.apache.hadoop.ozone.om.request.key.OMKeyPurgeRequest;
3843
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
3944
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
@@ -82,10 +87,15 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
8287
if (fromSnapshot != null) {
8388
OmSnapshotManager omSnapshotManager =
8489
((OmMetadataManagerImpl) omMetadataManager).getOzoneManager().getOmSnapshotManager();
85-
90+
IOzoneManagerLock lock = omMetadataManager.getLock();
91+
UUID fromSnapshotId = fromSnapshot.getSnapshotId();
92+
OMLockDetails lockDetails = lock.acquireReadLock(SNAPSHOT_DB_CONTENT_LOCK, fromSnapshotId.toString());
93+
if (!lockDetails.isLockAcquired()) {
94+
throw new OMException("Unable to acquire read lock on " + SNAPSHOT_DB_CONTENT_LOCK + " for snapshot: " +
95+
fromSnapshotId, OMException.ResultCodes.INTERNAL_ERROR);
96+
}
8697
try (UncheckedAutoCloseableSupplier<OmSnapshot> rcOmFromSnapshot =
87-
omSnapshotManager.getSnapshot(fromSnapshot.getSnapshotId())) {
88-
98+
omSnapshotManager.getSnapshot(fromSnapshotId)) {
8999
OmSnapshot fromOmSnapshot = rcOmFromSnapshot.get();
90100
DBStore fromSnapshotStore = fromOmSnapshot.getMetadataManager().getStore();
91101
// Init Batch Operation for snapshot db.
@@ -95,6 +105,8 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
95105
processKeysToUpdate(writeBatch, fromOmSnapshot.getMetadataManager());
96106
fromSnapshotStore.commitBatchOperation(writeBatch);
97107
}
108+
} finally {
109+
lock.releaseReadLock(SNAPSHOT_DB_CONTENT_LOCK, fromSnapshotId.toString());
98110
}
99111
omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, fromSnapshot.getTableKey(), fromSnapshot);
100112
} else {

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveTableKeysResponse.java‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@
1818
package org.apache.hadoop.ozone.om.response.snapshot;
1919

2020
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_INFO_TABLE;
21+
import static org.apache.hadoop.ozone.om.lock.FlatResource.SNAPSHOT_DB_CONTENT_LOCK;
2122
import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.createMergedRepeatedOmKeyInfoFromDeletedTableEntry;
2223

24+
import com.google.common.collect.Lists;
2325
import jakarta.annotation.Nonnull;
2426
import java.io.IOException;
2527
import java.util.List;
@@ -30,9 +32,12 @@
3032
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
3133
import org.apache.hadoop.ozone.om.OmSnapshot;
3234
import org.apache.hadoop.ozone.om.OmSnapshotManager;
35+
import org.apache.hadoop.ozone.om.exceptions.OMException;
3336
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
3437
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
3538
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
39+
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
40+
import org.apache.hadoop.ozone.om.lock.OMLockDetails;
3641
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
3742
import org.apache.hadoop.ozone.om.response.OMClientResponse;
3843
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
@@ -80,7 +85,15 @@ public OMSnapshotMoveTableKeysResponse(@Nonnull OMResponse omResponse) {
8085
protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException {
8186
OmSnapshotManager omSnapshotManager = ((OmMetadataManagerImpl) omMetadataManager)
8287
.getOzoneManager().getOmSnapshotManager();
83-
88+
IOzoneManagerLock lock = omMetadataManager.getLock();
89+
String[] fromSnapshotId = new String[] {fromSnapshot.getSnapshotId().toString()};
90+
String[] nextSnapshotId = nextSnapshot == null ? null : new String[] {nextSnapshot.getSnapshotId().toString()};
91+
List<String[]> snapshotIds = Lists.newArrayList(fromSnapshotId, nextSnapshotId);
92+
OMLockDetails lockDetails = lock.acquireReadLocks(SNAPSHOT_DB_CONTENT_LOCK, snapshotIds);
93+
if (!lockDetails.isLockAcquired()) {
94+
throw new OMException("Unable to acquire read lock on " + SNAPSHOT_DB_CONTENT_LOCK + " for snapshot: " +
95+
snapshotIds, OMException.ResultCodes.INTERNAL_ERROR);
96+
}
8497
try (UncheckedAutoCloseableSupplier<OmSnapshot> rcOmFromSnapshot =
8598
omSnapshotManager.getSnapshot(fromSnapshot.getSnapshotId())) {
8699

@@ -113,6 +126,8 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation
113126
fromSnapshotStore.getDb().flushWal(true);
114127
fromSnapshotStore.getDb().flush();
115128
}
129+
} finally {
130+
lock.releaseReadLocks(SNAPSHOT_DB_CONTENT_LOCK, snapshotIds);
116131
}
117132

118133
// Flush snapshot info to rocksDB.

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ public SnapshotDeletingService(long interval, long serviceTimeout,
117117
OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK,
118118
OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT);
119119
IOzoneManagerLock lock = getOzoneManager().getMetadataManager().getLock();
120-
this.snapshotIdLocks = new MultiSnapshotLocks(lock, SNAPSHOT_GC_LOCK, true);
120+
this.snapshotIdLocks = new MultiSnapshotLocks(lock, SNAPSHOT_GC_LOCK, true, 2);
121121
this.lockIds = new ArrayList<>(2);
122122
}
123123

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/MultiSnapshotLocks.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.hadoop.ozone.om.snapshot;
1919

20+
import com.google.common.annotations.VisibleForTesting;
2021
import java.util.ArrayList;
2122
import java.util.Arrays;
2223
import java.util.Collection;
@@ -39,11 +40,16 @@ public class MultiSnapshotLocks {
3940
private final boolean writeLock;
4041
private OMLockDetails lockDetails;
4142

43+
@VisibleForTesting
4244
public MultiSnapshotLocks(IOzoneManagerLock lock, Resource resource, boolean writeLock) {
45+
this(lock, resource, writeLock, 0);
46+
}
47+
48+
public MultiSnapshotLocks(IOzoneManagerLock lock, Resource resource, boolean writeLock, int maxNumberOfLocks) {
4349
this.writeLock = writeLock;
4450
this.resource = resource;
4551
this.lock = lock;
46-
this.objectLocks = new ArrayList<>();
52+
this.objectLocks = new ArrayList<>(maxNumberOfLocks);
4753
this.lockDetails = OMLockDetails.EMPTY_DETAILS_LOCK_NOT_ACQUIRED;
4854
}
4955

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableFilter.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,8 @@ public ReclaimableFilter(
9090
this.omSnapshotManager = omSnapshotManager;
9191
this.currentSnapshotInfo = currentSnapshotInfo;
9292
this.snapshotChainManager = snapshotChainManager;
93-
this.snapshotIdLocks = new MultiSnapshotLocks(lock, SNAPSHOT_GC_LOCK, false);
93+
this.snapshotIdLocks = new MultiSnapshotLocks(lock, SNAPSHOT_GC_LOCK, false,
94+
numberOfPreviousSnapshotsFromChain + 1);
9495
this.keyManager = keyManager;
9596
this.numberOfPreviousSnapshotsFromChain = numberOfPreviousSnapshotsFromChain;
9697
this.previousOmSnapshots = new ArrayList<>(numberOfPreviousSnapshotsFromChain);

‎hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.hadoop.ozone.om.request.key;
1919

2020
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
21+
import static org.apache.hadoop.ozone.om.lock.FlatResource.SNAPSHOT_DB_CONTENT_LOCK;
2122
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource.BUCKET_LOCK;
2223
import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getOmKeyInfo;
2324
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -32,6 +33,7 @@
3233
import static org.mockito.Mockito.spy;
3334
import static org.mockito.Mockito.when;
3435

36+
import com.google.common.collect.Lists;
3537
import jakarta.annotation.Nonnull;
3638
import java.io.IOException;
3739
import java.util.ArrayList;
@@ -433,7 +435,24 @@ public void testDirectoryPurge(boolean fromSnapshot, boolean purgeDirectory, int
433435
OMDirectoriesPurgeRequestWithFSO omKeyPurgeRequest = new OMDirectoriesPurgeRequestWithFSO(preExecutedRequest);
434436
OMDirectoriesPurgeResponseWithFSO omClientResponse = (OMDirectoriesPurgeResponseWithFSO) omKeyPurgeRequest
435437
.validateAndUpdateCache(ozoneManager, 100L);
438+
439+
IOzoneManagerLock lock = spy(omMetadataManager.getLock());
440+
when(omMetadataManager.getLock()).thenReturn(lock);
441+
List<String> locks = Lists.newArrayList();
442+
doAnswer(i -> {
443+
locks.add(i.getArgument(1));
444+
return i.callRealMethod();
445+
}).when(lock).acquireReadLock(eq(SNAPSHOT_DB_CONTENT_LOCK), anyString());
446+
447+
List<String> snapshotIds;
448+
if (fromSnapshot) {
449+
snapshotIds = Collections.singletonList(snapshotInfo.getSnapshotId().toString());
450+
} else {
451+
snapshotIds = Collections.emptyList();
452+
}
453+
436454
performBatchOperationCommit(omClientResponse);
455+
assertEquals(snapshotIds, locks);
437456
OmBucketInfo updatedBucketInfo = purgeDirectory || numberOfSubEntries > 0 ?
438457
omMetadataManager.getBucketTable().getSkipCache(bucketKey) : omMetadataManager.getBucketTable().get(bucketKey);
439458
long currentSnapshotUsedNamespace = updatedBucketInfo.getSnapshotUsedNamespace();

‎hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,21 @@
1717

1818
package org.apache.hadoop.ozone.om.request.key;
1919

20+
import static org.apache.hadoop.ozone.om.lock.FlatResource.SNAPSHOT_DB_CONTENT_LOCK;
2021
import static org.junit.jupiter.api.Assertions.assertEquals;
2122
import static org.junit.jupiter.api.Assertions.assertFalse;
2223
import static org.junit.jupiter.api.Assertions.assertNotEquals;
2324
import static org.junit.jupiter.api.Assertions.assertTrue;
25+
import static org.mockito.ArgumentMatchers.anyString;
26+
import static org.mockito.ArgumentMatchers.eq;
27+
import static org.mockito.Mockito.doAnswer;
28+
import static org.mockito.Mockito.spy;
2429
import static org.mockito.Mockito.when;
2530

31+
import com.google.common.collect.Lists;
2632
import java.io.IOException;
2733
import java.util.ArrayList;
34+
import java.util.Collections;
2835
import java.util.List;
2936
import java.util.UUID;
3037
import org.apache.commons.lang3.tuple.Pair;
@@ -35,6 +42,7 @@
3542
import org.apache.hadoop.ozone.om.OmSnapshot;
3643
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
3744
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
45+
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
3846
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
3947
import org.apache.hadoop.ozone.om.response.key.OMKeyPurgeResponse;
4048
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeletedKeys;
@@ -186,7 +194,6 @@ public void testKeyPurgeInSnapshot() throws Exception {
186194
.thenReturn(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE));
187195
// Create and Delete keys. The keys should be moved to DeletedKeys table
188196
Pair<List<String>, List<String>> deleteKeysAndRenamedEntry = createAndDeleteKeysAndRenamedEntry(1, null);
189-
190197
SnapshotInfo snapInfo = createSnapshot("snap1");
191198
assertEquals(snapInfo.getLastTransactionInfo(),
192199
TransactionInfo.valueOf(TransactionInfo.getTermIndex(1L)).toByteString());
@@ -235,6 +242,14 @@ public void testKeyPurgeInSnapshot() throws Exception {
235242
.setStatus(Status.OK)
236243
.build();
237244

245+
IOzoneManagerLock lock = spy(omMetadataManager.getLock());
246+
when(omMetadataManager.getLock()).thenReturn(lock);
247+
List<String> locks = Lists.newArrayList();
248+
doAnswer(i -> {
249+
locks.add(i.getArgument(1));
250+
return i.callRealMethod();
251+
}).when(lock).acquireReadLock(eq(SNAPSHOT_DB_CONTENT_LOCK), anyString());
252+
List<String> snapshotIds = Collections.singletonList(snapInfo.getSnapshotId().toString());
238253
try (BatchOperation batchOperation =
239254
omMetadataManager.getStore().initBatchOperation()) {
240255

@@ -245,6 +260,7 @@ public void testKeyPurgeInSnapshot() throws Exception {
245260
// Do manual commit and see whether addToBatch is successful or not.
246261
omMetadataManager.getStore().commitBatchOperation(batchOperation);
247262
}
263+
assertEquals(snapshotIds, locks);
248264
snapshotInfoOnDisk = omMetadataManager.getSnapshotInfoTable().getSkipCache(snapInfo.getTableKey());
249265
assertEquals(snapshotInfoOnDisk, snapInfo);
250266
// The keys should not exist in the DeletedKeys table

0 commit comments

Comments
 (0)