add rule for state machine ready

Author: sumitagrawl

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java

@@ -223,6 +223,11 @@ public final class SCMEvents {
   public static final TypedEvent<ContainerID>
       RECONCILE_CONTAINER = new TypedEvent<>(ContainerID.class, "Reconcile_Container");
 
+  /**
+   * This event will be triggered from SCM State Machine when ready.
+   */
+  public static final TypedEvent<Boolean> STATEMACHINE_READY = new TypedEvent<>(Boolean.class, "STATEMACHINE_READY");
+
   /**
    * Private Ctor. Never Constructed.
    */

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java

@@ -361,15 +361,17 @@ public void notifyTermIndexUpdated(long term, long index) {
       // 1. Refresh Safemode rules state.
       // 2. Start DN Rpc server.
       if (!refreshedAfterLeaderReady.get()) {
-        scm.getScmSafeModeManager().refresh();
-        scm.getDatanodeProtocolServer().start();
-
         refreshedAfterLeaderReady.set(true);
+        scm.getScmSafeModeManager().refreshAndValidate();
       }
       currentLeaderTerm.set(-1L);
     }
   }
 
+  public boolean isRefreshedAfterLeaderReady() {
+    return refreshedAfterLeaderReady.get();
+  }
+
   @Override
   public void notifyLeaderReady() {
     if (!isInitialized) {

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java

@@ -22,8 +22,11 @@
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
 import org.apache.hadoop.hdds.scm.ha.SCMContext;
+import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
 import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.hdds.server.events.EventQueue;
 
 /**
@@ -79,6 +82,16 @@ private void loadRules(SCMSafeModeManager safeModeManager) {
 
     preCheckRules.add(datanodeRule);
 
+    OzoneStorageContainerManager ozoneScm = scmContext.getScm();
+    if (ozoneScm instanceof StorageContainerManager) {
+      StorageContainerManager scm = (StorageContainerManager) ozoneScm;
+      SCMHAManager scmHAManager = scm.getScmHAManager();
+      if (scmHAManager != null && scmHAManager.getRatisServer() != null) {
+        preCheckRules.add(new StateMachineReadyRule(eventQueue, safeModeManager,
+            scmHAManager.getRatisServer().getSCMStateMachine()));
+      }
+    }
+
     if (pipelineManager != null) {
       safeModeRules.add(new HealthyPipelineSafeModeRule(eventQueue, pipelineManager,
           safeModeManager, config, scmContext, nodeManager));

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/StateMachineReadyRule.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.safemode;
+
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.ha.SCMStateMachine;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.server.events.TypedEvent;
+
+/**
+ * Class defining Safe mode exit when SCM State Machine is ready, ie all transaction is applied
+ */
+public class StateMachineReadyRule extends SafeModeExitRule<Boolean> {
+  private SCMStateMachine scmStateMachine = null;
+
+  public StateMachineReadyRule(EventQueue eventQueue, SCMSafeModeManager manager, SCMStateMachine scmStateMachine) {
+    super(manager, eventQueue);
+    this.scmStateMachine = scmStateMachine;
+  }
+
+  @Override
+  protected TypedEvent<Boolean> getEventType() {
+    return SCMEvents.STATEMACHINE_READY;
+  }
+
+  @Override
+  protected boolean validate() {
+    if (null != scmStateMachine) {
+      return scmStateMachine.isRefreshedAfterLeaderReady();
+    }
+    // if no HA, always return true.
+    return true;
+  }
+
+  @Override
+  protected void process(Boolean ignored) {
+  }
+
+  @Override
+  protected void cleanup() {
+  }
+
+  @Override
+  public String getStatusText() {
+    return String.format("Refreshed SCM State Machine after leader ready: %s",
+        scmStateMachine != null ? scmStateMachine.isRefreshedAfterLeaderReady() : "NA");
+  }
+
+  @Override
+  public void refresh(boolean forceRefresh) {
+    // Do nothing.
+  }
+}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java

@@ -1530,10 +1530,8 @@ public void start() throws IOException {
     }
     getBlockProtocolServer().start();
 
-    // If HA is enabled, start datanode protocol server once leader is ready.
-    if (!scmStorageConfig.isSCMHAEnabled()) {
-      getDatanodeProtocolServer().start();
-    }
+    // start datanode protocol server
+    getDatanodeProtocolServer().start();
     if (getSecurityProtocolServer() != null) {
       getSecurityProtocolServer().start();
       persistSCMCertificates();