From c0f26fb46c965dc784057ce94c70d5b664fc9893 Mon Sep 17 00:00:00 2001 From: Priyesh Karatha Date: Wed, 29 Oct 2025 11:48:20 +0530 Subject: [PATCH 1/2] HDDS-13852. Remove dependency on HDDSLayout for OM to SCM request --- .../org/apache/hadoop/hdds/scm/ScmInfo.java | 16 ++------------- ...ocationProtocolClientSideTranslatorPB.java | 20 +++---------------- ...ocationProtocolClientSideTranslatorPB.java | 5 +---- .../hadoop/ozone/common/BlockGroup.java | 16 --------------- ...ocationProtocolClientSideTranslatorPB.java | 2 -- ...ocationProtocolServerSideTranslatorPB.java | 18 +++++------------ ...ocationProtocolServerSideTranslatorPB.java | 1 - .../scm/server/SCMBlockProtocolServer.java | 3 +-- .../scm/server/SCMClientProtocolServer.java | 3 +-- .../hdds/scm/TestStorageContainerManager.java | 1 - .../TestDNDataDistributionFinalization.java | 9 +++++---- .../TestScmDataDistributionFinalization.java | 12 +++++------ .../om/service/TestBlockDeletionService.java | 6 ++++-- 13 files changed, 28 insertions(+), 84 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmInfo.java index 24bd67759d0f..93e1f8c1e018 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmInfo.java @@ -29,7 +29,6 @@ public final class ScmInfo { private final String clusterId; private final String scmId; private final List peerRoles; - private final int metadataLayoutVersion; /** * Builder for ScmInfo. @@ -38,7 +37,6 @@ public static class Builder { private String clusterId; private String scmId; private final List peerRoles; - private int metadataLayoutVersion; public Builder() { peerRoles = new ArrayList<>(); @@ -74,21 +72,15 @@ public Builder setPeerRoles(List roles) { return this; } - public Builder setMetaDataLayoutVersion(int version) { - this.metadataLayoutVersion = version; - return this; - } - public ScmInfo build() { - return new ScmInfo(clusterId, scmId, peerRoles, metadataLayoutVersion); + return new ScmInfo(clusterId, scmId, peerRoles); } } - private ScmInfo(String clusterId, String scmId, List peerRoles, int metadataLayoutVersion) { + private ScmInfo(String clusterId, String scmId, List peerRoles) { this.clusterId = clusterId; this.scmId = scmId; this.peerRoles = Collections.unmodifiableList(peerRoles); - this.metadataLayoutVersion = metadataLayoutVersion; } /** @@ -115,8 +107,4 @@ public List getPeerRoles() { return peerRoles; } - public int getMetaDataLayoutVersion() { - return metadataLayoutVersion; - } - } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java index f663828284ba..ac0c0cd15da8 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java @@ -65,7 +65,6 @@ import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.scm.proxy.SCMBlockLocationFailoverProxyProvider; import org.apache.hadoop.hdds.tracing.TracingUtil; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.io.retry.RetryProxy; import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolTranslator; @@ -249,10 +248,8 @@ public List allocateBlock( @Override public List deleteKeyBlocks( List keyBlocksInfoList) throws IOException { - boolean useDataDistribution = getScmInfoSafe().getMetaDataLayoutVersion() >= - HDDSLayoutFeature.DATA_DISTRIBUTION.layoutVersion(); List keyBlocksProto = keyBlocksInfoList.stream() - .map(blockGroup -> blockGroup.getProto(useDataDistribution)) + .map(BlockGroup::getProto) .collect(Collectors.toList()); List allResults = new ArrayList<>(); List batch = new ArrayList<>(); @@ -304,14 +301,6 @@ private List submitDeleteKeyBlocks(List batch return results; } - private synchronized ScmInfo getScmInfoSafe() throws IOException { - if (scmInfo == null || scmInfo.getMetaDataLayoutVersion() < - HDDSLayoutFeature.DATA_DISTRIBUTION.layoutVersion()) { - getScmInfo(); // refresh cached scmInfo - } - return scmInfo; - } - /** * Gets the cluster Id and Scm Id from SCM. * @return ScmInfo @@ -333,11 +322,8 @@ public ScmInfo getScmInfo() throws IOException { resp = wrappedResponse.getGetScmInfoResponse(); ScmInfo.Builder builder = new ScmInfo.Builder() .setClusterId(resp.getClusterId()) - .setScmId(resp.getScmId()) - .setMetaDataLayoutVersion(resp.hasMetaDataLayoutVersion() ? - resp.getMetaDataLayoutVersion() : HDDSLayoutFeature.INITIAL_VERSION.layoutVersion()); - scmInfo = builder.build(); - return scmInfo; + .setScmId(resp.getScmId()); + return builder.build(); } /** diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index 30d9622e0c6b..f5edfbd3e59a 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -137,7 +137,6 @@ import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider; import org.apache.hadoop.hdds.tracing.TracingUtil; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.io.retry.RetryProxy; import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolTranslator; @@ -780,9 +779,7 @@ public ScmInfo getScmInfo() throws IOException { ScmInfo.Builder builder = new ScmInfo.Builder() .setClusterId(resp.getClusterId()) .setScmId(resp.getScmId()) - .setPeerRoles(resp.getPeerRolesList()) - .setMetaDataLayoutVersion(resp.hasMetaDataLayoutVersion() ? - resp.getMetaDataLayoutVersion() : HDDSLayoutFeature.INITIAL_VERSION.layoutVersion()); + .setPeerRoles(resp.getPeerRolesList()); return builder.build(); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java index 5e4493e7ab9b..c5bf30e1f776 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java @@ -47,14 +47,6 @@ public String getGroupID() { } public KeyBlocks getProto() { - return getProtoForDeletedBlock(); - } - - public KeyBlocks getProto(boolean isIncludeBlockSize) { - return isIncludeBlockSize ? getProtoForDeletedBlock() : getProtoForBlockID(); - } - - private KeyBlocks getProtoForDeletedBlock() { KeyBlocks.Builder kbb = KeyBlocks.newBuilder(); for (DeletedBlock block : deletedBlocks) { ScmBlockLocationProtocolProtos.DeletedBlock deletedBlock = ScmBlockLocationProtocolProtos.DeletedBlock @@ -68,14 +60,6 @@ private KeyBlocks getProtoForDeletedBlock() { return kbb.setKey(groupID).build(); } - private KeyBlocks getProtoForBlockID() { - KeyBlocks.Builder kbb = KeyBlocks.newBuilder(); - for (DeletedBlock block : deletedBlocks) { - kbb.addBlocks(block.getBlockID().getProtobuf()); - } - return kbb.setKey(groupID).build(); - } - /** * Parses a KeyBlocks proto to a group of blocks. * @param proto KeyBlocks proto. diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocolPB/TestScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocolPB/TestScmBlockLocationProtocolClientSideTranslatorPB.java index c18e148e083c..1a8d01586421 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocolPB/TestScmBlockLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocolPB/TestScmBlockLocationProtocolClientSideTranslatorPB.java @@ -81,7 +81,6 @@ public void testGetScmInfo() throws IOException, ServiceException { // Verify assertEquals("test-cluster-id", result.getClusterId()); assertEquals("test-scm-id", result.getScmId()); - assertEquals(1, result.getMetaDataLayoutVersion()); verify(mockRpcProxy).send(any(), any(SCMBlockLocationRequest.class)); } @@ -108,7 +107,6 @@ public void testGetScmInfoWithoutMetaDataLayoutVersion() throws IOException, Ser // Verify assertEquals("test-cluster-id", result.getClusterId()); assertEquals("test-scm-id", result.getScmId()); - assertEquals(0, result.getMetaDataLayoutVersion()); verify(mockRpcProxy).send(any(), any(SCMBlockLocationRequest.class)); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java index 7ac65e4f30ef..0c522c2d877a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java @@ -145,18 +145,11 @@ private SCMBlockLocationResponse processMessage( request.getAllocateScmBlockRequest(), request.getVersion())); break; case DeleteScmKeyBlocks: - if (scm.getLayoutVersionManager().needsFinalization() && - !scm.getLayoutVersionManager() - .isAllowed(HDDSLayoutFeature.DATA_DISTRIBUTION) - ) { - boolean isRequestHasNewData = request.getDeleteScmKeyBlocksRequest().getKeyBlocksList().stream() - .anyMatch(keyBlocks -> keyBlocks.getDeletedBlocksCount() > 0); - - if (isRequestHasNewData) { - throw new SCMException("Cluster is not finalized yet, it is" - + " not enabled to to handle data distribution feature", - SCMException.ResultCodes.INTERNAL_ERROR); - } + boolean hasOldProtoType = request.getDeleteScmKeyBlocksRequest().getKeyBlocksList().stream() + .anyMatch(keyBlocks -> keyBlocks.getBlocksCount() > 0); + if (hasOldProtoType) { + throw new SCMException("Received delete request with an unsupported protocol version.", + SCMException.ResultCodes.INTERNAL_ERROR); } response.setDeleteScmKeyBlocksResponse( deleteScmKeyBlocks(request.getDeleteScmKeyBlocksRequest())); @@ -264,7 +257,6 @@ public HddsProtos.GetScmInfoResponseProto getScmInfo( return HddsProtos.GetScmInfoResponseProto.newBuilder() .setClusterId(scmInfo.getClusterId()) .setScmId(scmInfo.getScmId()) - .setMetaDataLayoutVersion(scmInfo.getMetaDataLayoutVersion()) .build(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java index 5d50347e9ca7..440d69b98575 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -1022,7 +1022,6 @@ public HddsProtos.GetScmInfoResponseProto getScmInfo( .setClusterId(scmInfo.getClusterId()) .setScmId(scmInfo.getScmId()) .addAllPeerRoles(scmInfo.getPeerRoles()) - .setMetaDataLayoutVersion(scmInfo.getMetaDataLayoutVersion()) .build(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 2dc862f208f9..dc982c063934 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -336,8 +336,7 @@ public ScmInfo getScmInfo() throws IOException { ScmInfo.Builder builder = new ScmInfo.Builder() .setClusterId(scm.getScmStorageConfig().getClusterID()) - .setScmId(scm.getScmStorageConfig().getScmId()) - .setMetaDataLayoutVersion(scm.getLayoutVersionManager().getMetadataLayoutVersion()); + .setScmId(scm.getScmStorageConfig().getScmId()); return builder.build(); } catch (Exception ex) { auditSuccess = false; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 365b47a2528f..46dced8b1ef5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -909,8 +909,7 @@ public ScmInfo getScmInfo() { new ScmInfo.Builder() .setClusterId(scm.getScmStorageConfig().getClusterID()) .setScmId(scm.getScmStorageConfig().getScmId()) - .setPeerRoles(scm.getScmHAManager().getRatisServer().getRatisRoles()) - .setMetaDataLayoutVersion(scm.getLayoutVersionManager().getMetadataLayoutVersion()); + .setPeerRoles(scm.getScmHAManager().getRatisServer().getRatisRoles()); ScmInfo info = builder.build(); AUDIT.logReadSuccess(buildAuditMessageForSuccess( SCMAction.GET_SCM_INFO, null)); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java index a1900fb88701..dee975c1f28a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java @@ -623,7 +623,6 @@ public void testScmInfo(@TempDir Path tempDir) throws Exception { ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo(); assertEquals(clusterId, scmInfo.getClusterId()); assertEquals(scmId, scmInfo.getScmId()); - assertTrue(scmInfo.getMetaDataLayoutVersion() >= 0); String expectedVersion = HddsVersionInfo.HDDS_VERSION_INFO.getVersion(); String actualVersion = scm.getSoftwareVersion(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java index b1af95ed4212..3b72004afa53 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java @@ -119,7 +119,8 @@ public void init(OzoneConfiguration conf) throws Exception { scmClient = cluster.getStorageContainerLocationClient(); cluster.waitForClusterToBeReady(); - assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(), scmClient.getScmInfo().getMetaDataLayoutVersion()); + assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(), + cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); // Create Volume and Bucket try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) { @@ -145,7 +146,7 @@ public void testDataDistributionUpgradeScenario() throws Exception { // Verify initial state - DATA_DISTRIBUTION should not be finalized yet assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(), - scmClient.getScmInfo().getMetaDataLayoutVersion()); + cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); // Create some data and delete operations to trigger pending deletion logic String keyName1 = "testKey1"; @@ -183,7 +184,7 @@ public void testDataDistributionUpgradeScenario() throws Exception { // Verify finalization completed assertEquals(HDDSLayoutFeature.DATA_DISTRIBUTION.layoutVersion(), - scmClient.getScmInfo().getMetaDataLayoutVersion()); + cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); // Create more data and deletions to test post-finalization behavior String keyName3 = "testKey3"; @@ -228,7 +229,7 @@ public void testMissingPendingDeleteMetadataRecalculation() throws Exception { TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); assertEquals(HDDSLayoutFeature.DATA_DISTRIBUTION.layoutVersion(), - scmClient.getScmInfo().getMetaDataLayoutVersion()); + cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); // Verify the system can handle scenarios where pendingDeleteBlockCount // might be missing and needs recalculation diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java index c43d881a93e1..d862cd227496 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java @@ -135,13 +135,14 @@ public void init(OzoneConfiguration conf, .setSCMConfigurator(configurator) .setNumDatanodes(NUM_DATANODES) .setDatanodeFactory(UniformDatanodesFactory.newBuilder() - .setLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion()) - .build()); + .setLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion()) + .build()); this.cluster = clusterBuilder.build(); scmClient = cluster.getStorageContainerLocationClient(); cluster.waitForClusterToBeReady(); - assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(), scmClient.getScmInfo().getMetaDataLayoutVersion()); + assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(), + cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); // Create Volume and Bucket try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) { @@ -190,7 +191,7 @@ public void testFinalizationEmptyClusterDataDistribution() throws Exception { // Make sure old leader has caught up and all SCMs have finalized. waitForScmsToFinalize(cluster.getStorageContainerManagersList()); assertEquals(HDDSLayoutFeature.DATA_DISTRIBUTION.layoutVersion(), - scmClient.getScmInfo().getMetaDataLayoutVersion()); + cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); TestHddsUpgradeUtils.testPostUpgradeConditionsSCM( cluster.getStorageContainerManagersList(), 0, NUM_DATANODES); @@ -329,8 +330,7 @@ public void testFinalizationNonEmptyClusterDataDistribution() throws Exception { // Make sure old leader has caught up and all SCMs have finalized. waitForScmsToFinalize(cluster.getStorageContainerManagersList()); assertEquals(HDDSLayoutFeature.DATA_DISTRIBUTION.layoutVersion(), - scmClient.getScmInfo().getMetaDataLayoutVersion()); - + cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); TestHddsUpgradeUtils.testPostUpgradeConditionsSCM( cluster.getStorageContainerManagersList(), 0, NUM_DATANODES); TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes( diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java index 1053d808f7de..918e0fd8726e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java @@ -112,7 +112,8 @@ public static void init() throws Exception { .build(); cluster.waitForClusterToBeReady(); scmClient = cluster.getStorageContainerLocationClient(); - assertEquals(HBASE_SUPPORT.ordinal(), scmClient.getScmInfo().getMetaDataLayoutVersion()); + assertEquals(HBASE_SUPPORT.ordinal(), + cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); metrics = cluster.getStorageContainerManager().getBlockProtocolServer().getMetrics(); OzoneClient ozoneClient = cluster.newClient(); @@ -161,7 +162,8 @@ public void testDeleteKeyQuotaWithUpgrade() throws Exception { }); finalizationFuture.get(); TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); - assertEquals(DATA_DISTRIBUTION.ordinal(), scmClient.getScmInfo().getMetaDataLayoutVersion()); + assertEquals(DATA_DISTRIBUTION.ordinal(), + cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); // POST-UPGRADE //Step 6: Repeat the same steps in pre-upgrade From 7756e057ab48d1634d4604dde8a2b1258c069df7 Mon Sep 17 00:00:00 2001 From: Priyesh Karatha Date: Wed, 29 Oct 2025 18:11:51 +0530 Subject: [PATCH 2/2] fixing integration test --- .../om/service/TestBlockDeletionService.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java index 918e0fd8726e..559be0043966 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java @@ -20,7 +20,6 @@ import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATA_DISTRIBUTION; import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.HBASE_SUPPORT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.common.BlockGroup.SIZE_NOT_AVAILABLE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.params.provider.Arguments.arguments; @@ -147,7 +146,7 @@ public void testDeleteKeyQuotaWithUpgrade() throws Exception { bucket.deleteKey(keyName); // Step 4: Verify deleteBlocks call and capture argument verify(spyManagerBefore, timeout(50000).atLeastOnce()).deleteBlocks(captor.capture()); - verifyAndAssertQuota(replicationConfig, captor, false); + verifyAndAssertQuota(replicationConfig, captor); GenericTestUtils.waitFor(() -> metrics.getDeleteKeySuccessBlocks() - initialSuccessBlocks == 1, 50, 1000); GenericTestUtils.waitFor(() -> metrics.getDeleteKeyFailedBlocks() - initialFailedBlocks == 0, 50, 1000); @@ -172,7 +171,7 @@ public void testDeleteKeyQuotaWithUpgrade() throws Exception { BlockManager spyManagerAfter = injectSpyBlockManager(cluster); bucket.deleteKey(keyName); verify(spyManagerAfter, timeout(50000).atLeastOnce()).deleteBlocks(captor.capture()); - verifyAndAssertQuota(replicationConfig, captor, true); + verifyAndAssertQuota(replicationConfig, captor); GenericTestUtils.waitFor(() -> metrics.getDeleteKeySuccessBlocks() - initialSuccessBlocks == 2, 50, 1000); GenericTestUtils.waitFor(() -> metrics.getDeleteKeyFailedBlocks() - initialFailedBlocks == 0, 50, 1000); } @@ -193,7 +192,7 @@ public void testDeleteKeyQuotaWithDifferentReplicationTypes(ReplicationConfig re bucket.deleteKey(keyName); // Step 4: Verify deleteBlocks call and capture argument verify(spyManagerBefore, timeout(50000).atLeastOnce()).deleteBlocks(captor.capture()); - verifyAndAssertQuota(replicationConfig, captor, true); + verifyAndAssertQuota(replicationConfig, captor); GenericTestUtils.waitFor(() -> metrics.getDeleteKeySuccessBlocks() - initialSuccessBlocks == 1, 50, 1000); GenericTestUtils.waitFor(() -> metrics.getDeleteKeyFailedBlocks() - initialFailedBlocks == 0, 50, 1000); } @@ -218,8 +217,7 @@ private BlockManager injectSpyBlockManager(MiniOzoneCluster miniOzoneCluster) th } private void verifyAndAssertQuota(ReplicationConfig replicationConfig, - ArgumentCaptor> captor, - boolean isIncludeBlockSize) throws IOException { + ArgumentCaptor> captor) throws IOException { int index = captor.getAllValues().size() - 1; List blockGroups = captor.getAllValues().get(index); @@ -232,8 +230,7 @@ private void verifyAndAssertQuota(ReplicationConfig replicationConfig, .mapToLong(DeletedBlock::getSize).sum(); assertEquals(1, blockGroups.get(0).getAllDeletedBlocks().size()); - assertEquals(isIncludeBlockSize ? - QuotaUtil.getReplicatedSize(KEY_SIZE, replicationConfig) : SIZE_NOT_AVAILABLE, totalUsedBytes); - assertEquals(isIncludeBlockSize ? KEY_SIZE : SIZE_NOT_AVAILABLE, totalUnreplicatedBytes); + assertEquals(QuotaUtil.getReplicatedSize(KEY_SIZE, replicationConfig), totalUsedBytes); + assertEquals(KEY_SIZE, totalUnreplicatedBytes); } }