HDDS-13852. Remove dependency on HDDSLayout for OM to SCM request

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmInfo.java

@@ -29,7 +29,6 @@ public final class ScmInfo {
   private final String clusterId;
   private final String scmId;
   private final List<String> peerRoles;
-  private final int metadataLayoutVersion;
 
   /**
    * Builder for ScmInfo.
@@ -38,7 +37,6 @@ public static class Builder {
     private String clusterId;
     private String scmId;
     private final List<String> peerRoles;
-    private int metadataLayoutVersion;
 
     public Builder() {
       peerRoles = new ArrayList<>();
@@ -74,21 +72,15 @@ public Builder setPeerRoles(List<String> roles) {
       return this;
     }
 
-    public Builder setMetaDataLayoutVersion(int version) {
-      this.metadataLayoutVersion = version;
-      return this;
-    }
-
     public ScmInfo build() {
-      return new ScmInfo(clusterId, scmId, peerRoles, metadataLayoutVersion);
+      return new ScmInfo(clusterId, scmId, peerRoles);
     }
   }
 
-  private ScmInfo(String clusterId, String scmId, List<String> peerRoles, int metadataLayoutVersion) {
+  private ScmInfo(String clusterId, String scmId, List<String> peerRoles) {
     this.clusterId = clusterId;
     this.scmId = scmId;
     this.peerRoles = Collections.unmodifiableList(peerRoles);
-    this.metadataLayoutVersion = metadataLayoutVersion;
   }
 
   /**
@@ -115,8 +107,4 @@ public List<String> getPeerRoles() {
     return peerRoles;
   }
 
-  public int getMetaDataLayoutVersion() {
-    return metadataLayoutVersion;
-  }
-
 }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java

@@ -65,7 +65,6 @@
 import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
 import org.apache.hadoop.hdds.scm.proxy.SCMBlockLocationFailoverProxyProvider;
 import org.apache.hadoop.hdds.tracing.TracingUtil;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
 import org.apache.hadoop.io.retry.RetryProxy;
 import org.apache.hadoop.ipc.ProtobufHelper;
 import org.apache.hadoop.ipc.ProtocolTranslator;
@@ -249,10 +248,8 @@ public List<AllocatedBlock> allocateBlock(
   @Override
   public List<DeleteBlockGroupResult> deleteKeyBlocks(
       List<BlockGroup> keyBlocksInfoList) throws IOException {
-    boolean useDataDistribution = getScmInfoSafe().getMetaDataLayoutVersion() >=
-        HDDSLayoutFeature.DATA_DISTRIBUTION.layoutVersion();
     List<KeyBlocks> keyBlocksProto = keyBlocksInfoList.stream()
-        .map(blockGroup -> blockGroup.getProto(useDataDistribution))
+        .map(BlockGroup::getProto)
         .collect(Collectors.toList());
     List<DeleteBlockGroupResult> allResults = new ArrayList<>();
     List<KeyBlocks> batch = new ArrayList<>();
@@ -304,14 +301,6 @@ private List<DeleteBlockGroupResult> submitDeleteKeyBlocks(List<KeyBlocks> batch
     return results;
   }
 
-  private synchronized ScmInfo getScmInfoSafe() throws IOException {
-    if (scmInfo == null || scmInfo.getMetaDataLayoutVersion() <
-        HDDSLayoutFeature.DATA_DISTRIBUTION.layoutVersion()) {
-      getScmInfo(); // refresh cached scmInfo
-    }
-    return scmInfo;
-  }
-
   /**
    * Gets the cluster Id and Scm Id from SCM.
    * @return ScmInfo
@@ -333,11 +322,8 @@ public ScmInfo getScmInfo() throws IOException {
     resp = wrappedResponse.getGetScmInfoResponse();
     ScmInfo.Builder builder = new ScmInfo.Builder()
         .setClusterId(resp.getClusterId())
-        .setScmId(resp.getScmId())
-        .setMetaDataLayoutVersion(resp.hasMetaDataLayoutVersion() ?
-            resp.getMetaDataLayoutVersion() : HDDSLayoutFeature.INITIAL_VERSION.layoutVersion());
-    scmInfo = builder.build();
-    return scmInfo;
+        .setScmId(resp.getScmId());
+    return  builder.build();
   }
 
   /**

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java

@@ -137,7 +137,6 @@
 import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
 import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider;
 import org.apache.hadoop.hdds.tracing.TracingUtil;
-import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
 import org.apache.hadoop.io.retry.RetryProxy;
 import org.apache.hadoop.ipc.ProtobufHelper;
 import org.apache.hadoop.ipc.ProtocolTranslator;
@@ -780,9 +779,7 @@ public ScmInfo getScmInfo() throws IOException {
     ScmInfo.Builder builder = new ScmInfo.Builder()
         .setClusterId(resp.getClusterId())
         .setScmId(resp.getScmId())
-        .setPeerRoles(resp.getPeerRolesList())
-        .setMetaDataLayoutVersion(resp.hasMetaDataLayoutVersion() ?
-            resp.getMetaDataLayoutVersion() : HDDSLayoutFeature.INITIAL_VERSION.layoutVersion());
+        .setPeerRoles(resp.getPeerRolesList());
     return builder.build();
   }
 

hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java

@@ -47,14 +47,6 @@ public String getGroupID() {
   }
 
   public KeyBlocks getProto() {
-    return getProtoForDeletedBlock();
-  }
-
-  public KeyBlocks getProto(boolean isIncludeBlockSize) {
-    return isIncludeBlockSize ? getProtoForDeletedBlock() : getProtoForBlockID();
-  }
-
-  private KeyBlocks getProtoForDeletedBlock() {
     KeyBlocks.Builder kbb = KeyBlocks.newBuilder();
     for (DeletedBlock block : deletedBlocks) {
       ScmBlockLocationProtocolProtos.DeletedBlock deletedBlock = ScmBlockLocationProtocolProtos.DeletedBlock
@@ -68,14 +60,6 @@ private KeyBlocks getProtoForDeletedBlock() {
     return kbb.setKey(groupID).build();
   }
 
-  private KeyBlocks getProtoForBlockID() {
-    KeyBlocks.Builder kbb = KeyBlocks.newBuilder();
-    for (DeletedBlock block : deletedBlocks) {
-      kbb.addBlocks(block.getBlockID().getProtobuf());
-    }
-    return kbb.setKey(groupID).build();
-  }
-
   /**
    * Parses a KeyBlocks proto to a group of blocks.
    * @param proto KeyBlocks proto.

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/protocolPB/TestScmBlockLocationProtocolClientSideTranslatorPB.java

@@ -81,7 +81,6 @@ public void testGetScmInfo() throws IOException, ServiceException {
     // Verify
     assertEquals("test-cluster-id", result.getClusterId());
     assertEquals("test-scm-id", result.getScmId());
-    assertEquals(1, result.getMetaDataLayoutVersion());
     verify(mockRpcProxy).send(any(), any(SCMBlockLocationRequest.class));
   }
 
@@ -108,7 +107,6 @@ public void testGetScmInfoWithoutMetaDataLayoutVersion() throws IOException, Ser
     // Verify
     assertEquals("test-cluster-id", result.getClusterId());
     assertEquals("test-scm-id", result.getScmId());
-    assertEquals(0, result.getMetaDataLayoutVersion());
     verify(mockRpcProxy).send(any(), any(SCMBlockLocationRequest.class));
   }
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocolServerSideTranslatorPB.java

@@ -145,18 +145,11 @@ private SCMBlockLocationResponse processMessage(
             request.getAllocateScmBlockRequest(), request.getVersion()));
         break;
       case DeleteScmKeyBlocks:
-        if (scm.getLayoutVersionManager().needsFinalization() &&
-            !scm.getLayoutVersionManager()
-                .isAllowed(HDDSLayoutFeature.DATA_DISTRIBUTION)
-        ) {
-          boolean isRequestHasNewData = request.getDeleteScmKeyBlocksRequest().getKeyBlocksList().stream()
-              .anyMatch(keyBlocks -> keyBlocks.getDeletedBlocksCount() > 0);
-
-          if (isRequestHasNewData) {
-            throw new SCMException("Cluster is not finalized yet, it is"
-                + " not enabled to to handle data distribution feature",
-                SCMException.ResultCodes.INTERNAL_ERROR);
-          }
+        boolean hasOldProtoType = request.getDeleteScmKeyBlocksRequest().getKeyBlocksList().stream()
+            .anyMatch(keyBlocks -> keyBlocks.getBlocksCount() > 0);
+        if (hasOldProtoType) {
+          throw new SCMException("Received delete request with an unsupported protocol version.",
+              SCMException.ResultCodes.INTERNAL_ERROR);
         }
         response.setDeleteScmKeyBlocksResponse(
             deleteScmKeyBlocks(request.getDeleteScmKeyBlocksRequest()));
@@ -264,7 +257,6 @@ public HddsProtos.GetScmInfoResponseProto getScmInfo(
     return HddsProtos.GetScmInfoResponseProto.newBuilder()
         .setClusterId(scmInfo.getClusterId())
         .setScmId(scmInfo.getScmId())
-        .setMetaDataLayoutVersion(scmInfo.getMetaDataLayoutVersion())
         .build();
   }
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java

@@ -1022,7 +1022,6 @@ public HddsProtos.GetScmInfoResponseProto getScmInfo(
         .setClusterId(scmInfo.getClusterId())
         .setScmId(scmInfo.getScmId())
         .addAllPeerRoles(scmInfo.getPeerRoles())
-        .setMetaDataLayoutVersion(scmInfo.getMetaDataLayoutVersion())
         .build();
   }
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java

@@ -336,8 +336,7 @@ public ScmInfo getScmInfo() throws IOException {
       ScmInfo.Builder builder =
           new ScmInfo.Builder()
               .setClusterId(scm.getScmStorageConfig().getClusterID())
-              .setScmId(scm.getScmStorageConfig().getScmId())
-              .setMetaDataLayoutVersion(scm.getLayoutVersionManager().getMetadataLayoutVersion());
+              .setScmId(scm.getScmStorageConfig().getScmId());
       return builder.build();
     } catch (Exception ex) {
       auditSuccess = false;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java

@@ -909,8 +909,7 @@ public ScmInfo getScmInfo() {
           new ScmInfo.Builder()
               .setClusterId(scm.getScmStorageConfig().getClusterID())
               .setScmId(scm.getScmStorageConfig().getScmId())
-              .setPeerRoles(scm.getScmHAManager().getRatisServer().getRatisRoles())
-              .setMetaDataLayoutVersion(scm.getLayoutVersionManager().getMetadataLayoutVersion());
+              .setPeerRoles(scm.getScmHAManager().getRatisServer().getRatisRoles());
       ScmInfo info = builder.build();
       AUDIT.logReadSuccess(buildAuditMessageForSuccess(
             SCMAction.GET_SCM_INFO, null));

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java

@@ -623,7 +623,6 @@ public void testScmInfo(@TempDir Path tempDir) throws Exception {
       ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
       assertEquals(clusterId, scmInfo.getClusterId());
       assertEquals(scmId, scmInfo.getScmId());
-      assertTrue(scmInfo.getMetaDataLayoutVersion() >= 0);
 
       String expectedVersion = HddsVersionInfo.HDDS_VERSION_INFO.getVersion();
       String actualVersion = scm.getSoftwareVersion();

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java

@@ -119,7 +119,8 @@ public void init(OzoneConfiguration conf) throws Exception {
 
     scmClient = cluster.getStorageContainerLocationClient();
     cluster.waitForClusterToBeReady();
-    assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(), scmClient.getScmInfo().getMetaDataLayoutVersion());
+    assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(),
+        cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
 
     // Create Volume and Bucket
     try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) {
@@ -145,7 +146,7 @@ public void testDataDistributionUpgradeScenario() throws Exception {
 
     // Verify initial state - DATA_DISTRIBUTION should not be finalized yet
     assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(),
-        scmClient.getScmInfo().getMetaDataLayoutVersion());
+        cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
 
     // Create some data and delete operations to trigger pending deletion logic
     String keyName1 = "testKey1";
@@ -183,7 +184,7 @@ public void testDataDistributionUpgradeScenario() throws Exception {
 
     // Verify finalization completed
     assertEquals(HDDSLayoutFeature.DATA_DISTRIBUTION.layoutVersion(),
-        scmClient.getScmInfo().getMetaDataLayoutVersion());
+        cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
 
     // Create more data and deletions to test post-finalization behavior
     String keyName3 = "testKey3";
@@ -228,7 +229,7 @@ public void testMissingPendingDeleteMetadataRecalculation() throws Exception {
     TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
 
     assertEquals(HDDSLayoutFeature.DATA_DISTRIBUTION.layoutVersion(),
-        scmClient.getScmInfo().getMetaDataLayoutVersion());
+        cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
 
     // Verify the system can handle scenarios where pendingDeleteBlockCount
     // might be missing and needs recalculation

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java

@@ -135,13 +135,14 @@ public void init(OzoneConfiguration conf,
         .setSCMConfigurator(configurator)
         .setNumDatanodes(NUM_DATANODES)
         .setDatanodeFactory(UniformDatanodesFactory.newBuilder()
-            .setLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion())
-            .build());
+        .setLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion())
+        .build());
     this.cluster = clusterBuilder.build();
 
     scmClient = cluster.getStorageContainerLocationClient();
     cluster.waitForClusterToBeReady();
-    assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(), scmClient.getScmInfo().getMetaDataLayoutVersion());
+    assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(),
+        cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
 
     // Create Volume and Bucket
     try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) {
@@ -190,7 +191,7 @@ public void testFinalizationEmptyClusterDataDistribution() throws Exception {
     // Make sure old leader has caught up and all SCMs have finalized.
     waitForScmsToFinalize(cluster.getStorageContainerManagersList());
     assertEquals(HDDSLayoutFeature.DATA_DISTRIBUTION.layoutVersion(),
-        scmClient.getScmInfo().getMetaDataLayoutVersion());
+        cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
 
     TestHddsUpgradeUtils.testPostUpgradeConditionsSCM(
         cluster.getStorageContainerManagersList(), 0, NUM_DATANODES);
@@ -329,8 +330,7 @@ public void testFinalizationNonEmptyClusterDataDistribution() throws Exception {
     // Make sure old leader has caught up and all SCMs have finalized.
     waitForScmsToFinalize(cluster.getStorageContainerManagersList());
     assertEquals(HDDSLayoutFeature.DATA_DISTRIBUTION.layoutVersion(),
-        scmClient.getScmInfo().getMetaDataLayoutVersion());
-
+        cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
     TestHddsUpgradeUtils.testPostUpgradeConditionsSCM(
         cluster.getStorageContainerManagersList(), 0, NUM_DATANODES);
     TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes(

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java

@@ -20,7 +20,6 @@
 import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.DATA_DISTRIBUTION;
 import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.HBASE_SUPPORT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
-import static org.apache.hadoop.ozone.common.BlockGroup.SIZE_NOT_AVAILABLE;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.fail;
 import static org.junit.jupiter.params.provider.Arguments.arguments;
@@ -112,7 +111,8 @@ public static void init() throws Exception {
         .build();
     cluster.waitForClusterToBeReady();
     scmClient = cluster.getStorageContainerLocationClient();
-    assertEquals(HBASE_SUPPORT.ordinal(), scmClient.getScmInfo().getMetaDataLayoutVersion());
+    assertEquals(HBASE_SUPPORT.ordinal(),
+        cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
     metrics = cluster.getStorageContainerManager().getBlockProtocolServer().getMetrics();
 
     OzoneClient ozoneClient = cluster.newClient();
@@ -146,7 +146,7 @@ public void testDeleteKeyQuotaWithUpgrade() throws Exception {
     bucket.deleteKey(keyName);
     // Step 4: Verify deleteBlocks call and capture argument
     verify(spyManagerBefore, timeout(50000).atLeastOnce()).deleteBlocks(captor.capture());
-    verifyAndAssertQuota(replicationConfig, captor, false);
+    verifyAndAssertQuota(replicationConfig, captor);
     GenericTestUtils.waitFor(() -> metrics.getDeleteKeySuccessBlocks() - initialSuccessBlocks == 1, 50, 1000);
     GenericTestUtils.waitFor(() -> metrics.getDeleteKeyFailedBlocks() - initialFailedBlocks == 0, 50, 1000);
 
@@ -161,7 +161,8 @@ public void testDeleteKeyQuotaWithUpgrade() throws Exception {
     });
     finalizationFuture.get();
     TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
-    assertEquals(DATA_DISTRIBUTION.ordinal(), scmClient.getScmInfo().getMetaDataLayoutVersion());
+    assertEquals(DATA_DISTRIBUTION.ordinal(),
+        cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
 
     // POST-UPGRADE
     //Step 6: Repeat the same steps in pre-upgrade
@@ -170,7 +171,7 @@ public void testDeleteKeyQuotaWithUpgrade() throws Exception {
     BlockManager spyManagerAfter = injectSpyBlockManager(cluster);
     bucket.deleteKey(keyName);
     verify(spyManagerAfter, timeout(50000).atLeastOnce()).deleteBlocks(captor.capture());
-    verifyAndAssertQuota(replicationConfig, captor, true);
+    verifyAndAssertQuota(replicationConfig, captor);
     GenericTestUtils.waitFor(() -> metrics.getDeleteKeySuccessBlocks() - initialSuccessBlocks == 2, 50, 1000);
     GenericTestUtils.waitFor(() -> metrics.getDeleteKeyFailedBlocks() - initialFailedBlocks == 0, 50, 1000);
   }
@@ -191,7 +192,7 @@ public void testDeleteKeyQuotaWithDifferentReplicationTypes(ReplicationConfig re
     bucket.deleteKey(keyName);
     // Step 4: Verify deleteBlocks call and capture argument
     verify(spyManagerBefore, timeout(50000).atLeastOnce()).deleteBlocks(captor.capture());
-    verifyAndAssertQuota(replicationConfig, captor, true);
+    verifyAndAssertQuota(replicationConfig, captor);
     GenericTestUtils.waitFor(() -> metrics.getDeleteKeySuccessBlocks() - initialSuccessBlocks == 1, 50, 1000);
     GenericTestUtils.waitFor(() -> metrics.getDeleteKeyFailedBlocks() - initialFailedBlocks == 0, 50, 1000);
   }
@@ -216,8 +217,7 @@ private BlockManager injectSpyBlockManager(MiniOzoneCluster miniOzoneCluster) th
   }
 
   private void verifyAndAssertQuota(ReplicationConfig replicationConfig,
-                                    ArgumentCaptor<List<BlockGroup>> captor,
-                                    boolean isIncludeBlockSize) throws IOException {
+                                    ArgumentCaptor<List<BlockGroup>> captor) throws IOException {
     int index = captor.getAllValues().size() - 1;
     List<BlockGroup> blockGroups = captor.getAllValues().get(index);
 
@@ -230,8 +230,7 @@ private void verifyAndAssertQuota(ReplicationConfig replicationConfig,
         .mapToLong(DeletedBlock::getSize).sum();
 
     assertEquals(1, blockGroups.get(0).getAllDeletedBlocks().size());
-    assertEquals(isIncludeBlockSize ?
-        QuotaUtil.getReplicatedSize(KEY_SIZE, replicationConfig) : SIZE_NOT_AVAILABLE, totalUsedBytes);
-    assertEquals(isIncludeBlockSize ? KEY_SIZE : SIZE_NOT_AVAILABLE, totalUnreplicatedBytes);
+    assertEquals(QuotaUtil.getReplicatedSize(KEY_SIZE, replicationConfig), totalUsedBytes);
+    assertEquals(KEY_SIZE, totalUnreplicatedBytes);
   }
 }