fix(redis): 切换同步检查加入自己的心跳机制 #14397

Author: xiepaup

dbm-services/redis/db-tools/dbactuator/pkg/atomjobs/atomredis/redis_switch.go

@@ -929,7 +929,10 @@ func (job *RedisSwitchPreCheck) checkReplicationSync(newMasterConn *myredis.Redi
 	var masterTime, masterDbsize, slaveTime int64
 	oldMasterAddr := fmt.Sprintf("%s:%d", storagePair.MasterInfo.IP, storagePair.MasterInfo.Port)
 	newMasterAddr := fmt.Sprintf("%s:%d", storagePair.SlaveInfo.IP, storagePair.SlaveInfo.Port)
-
+	if err := job.trySetHeartBeatWhenDbmonStoped(storagePair); err != nil {
+		time.Sleep(time.Second * 2)
+		job.trySetHeartBeatWhenDbmonStoped(storagePair)
+	}
 	if err := newMasterConn.SelectDB(1); err != nil {
 		return fmt.Errorf("[%s] select db 1, exec cmd err:%+v", newMasterAddr, err)
 	}
@@ -976,6 +979,30 @@ func (job *RedisSwitchPreCheck) checkReplicationSync(newMasterConn *myredis.Redi
 	return nil
 }
 
+// setHeartBeatWhenDbmonStoped # here we just set on the master heartbeat:
+func (job *RedisSwitchPreCheck) trySetHeartBeatWhenDbmonStoped(storagePair InstanceSwitchParam) error {
+	oldMasterAddr := fmt.Sprintf("%s:%d", storagePair.MasterInfo.IP, storagePair.MasterInfo.Port)
+	oldMasterConn, err := myredis.NewRedisClientWithTimeout(oldMasterAddr,
+		job.params.ClusterMeta.StoragePassword, 0, job.params.ClusterMeta.ClusterType, time.Second*10)
+	if err != nil {
+		job.runtime.Logger.Warn("[%s] conntect 2 old master failed :%+v", oldMasterAddr, err)
+		return err
+	}
+	defer oldMasterConn.Close()
+
+	if err := oldMasterConn.SelectDB(1); err != nil {
+		job.runtime.Logger.Warn("[%s] select db 1 on  master failed :%+v", oldMasterAddr, err)
+		return err
+	}
+	job.runtime.Logger.Info("[%s] try reset dbmon heartbeat key [%s] value 2 current", oldMasterAddr, fmt.Sprintf("%s:time", oldMasterAddr))
+	rst := oldMasterConn.InstanceClient.Set(context.TODO(), fmt.Sprintf("%s:time", oldMasterAddr), time.Now().Unix(), 0)
+	if rst.Err() != nil {
+		job.runtime.Logger.Warn("[%s] set on old master failed :%+v", oldMasterAddr, err)
+		return err
+	}
+	return nil
+}
+
 // precheckStorageLogin make sure all todo switch redis can login
 func (job *RedisSwitchPreCheck) precheckStorageLogin() error {
 	wg := &sync.WaitGroup{}

dbm-ui/backend/flow/consts.py

@@ -57,10 +57,10 @@
 DEFAULT_REDIS_INSTANCE_DATABASES = 2
 DEFAULT_REDIS_CLUSTER_DATABASES = 1
 
-# 切换时， 默认允许多久心跳
-DEFAULT_MASTER_DIFF_TIME = 61
-# 切换时， 允许多少秒丢失
-DEFAULT_LAST_IO_SECOND_AGO = 100
+# 切换时， 默认允许多久心跳 61->11 @20251121
+DEFAULT_MASTER_DIFF_TIME = 11
+# 切换时， 允许多少秒丢失 100->30 @20251121
+DEFAULT_LAST_IO_SECOND_AGO = 30
 
 # 默认Riak端口
 DEFAULT_RIAK_PORT = 8087