fix(mysql): 分区查询优化及特殊场景处理 #8652

dbm-services/mysql/db-partition/assests/migrations/000008_init.down.sql

@@ -0,0 +1 @@
+SET NAMES utf8;

dbm-services/mysql/db-partition/assests/migrations/000008_init.up.sql

@@ -0,0 +1,9 @@
+SET NAMES utf8;
+CREATE TABLE `partition_customization_config` (
+    `id` int(11) NOT NULL AUTO_INCREMENT,
+    `bk_biz_id` int(11) NOT NULL,
+    `immute_domain` varchar(255) NOT NULL DEFAULT '',
+    `partition_column` varchar(255) NOT NULL DEFAULT '',
+    PRIMARY KEY (`id`),
+    KEY `bk_biz_id` (`bk_biz_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

dbm-services/mysql/db-partition/handler/handler.go

@@ -1,16 +1,17 @@
-// Package handler TODO
+// Package handler TODOG
 package handler
 
 import (
-	"dbm-services/mysql/db-partition/model"
-	"dbm-services/mysql/db-partition/monitor"
 	"errors"
 	"fmt"
 	"log/slog"
 	"net/http"
 	_ "runtime/debug" // debug TODO
 	"time"
 
+	"dbm-services/mysql/db-partition/model"
+	"dbm-services/mysql/db-partition/monitor"
+
 	cron_pkg "github.com/robfig/cron/v3"
 
 	"dbm-services/common/go-pubpkg/errno"
@@ -185,7 +186,7 @@ func DisablePartitionByCluster(r *gin.Context) {
 		SendResponse(r, err, nil)
 		return
 	}
-	slog.Info(fmt.Sprintf("ids: %v, operator: %s", input.Ids, input.Operator))
+	slog.Info(fmt.Sprintf("cluster_ids: %v, operator: %s", input.ClusterIds, input.Operator))
 	err := input.DisablePartitionConfigByCluster()
 	if err != nil {
 		slog.Error(err.Error())

dbm-services/mysql/db-partition/main.go

@@ -1,11 +1,12 @@
 package main
 
 import (
+	"net/http"
+	"os"
+
 	"dbm-services/mysql/db-partition/monitor"
 	"dbm-services/mysql/db-partition/service"
 	"dbm-services/mysql/db-partition/util"
-	"net/http"
-	"os"
 
 	"github.com/gin-gonic/gin"
 	"github.com/golang-migrate/migrate/v4"
@@ -82,4 +83,5 @@ func init() {
 	model.DB.Init()
 	model.InitClient()
 	model.InitBkRepo()
+	model.InitCustimazation()
 }

dbm-services/mysql/db-partition/model/init_custimazation.go

@@ -0,0 +1,46 @@
+/*
+ * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+ * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at https://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package model
+
+import (
+	"log/slog"
+
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+var CustimazationMap map[int64]string
+
+// Custimazation TODO
+type Custimazation struct {
+	id              int    `gorm:"column:id"`
+	BkBizId         int64  `json:"bk_biz_id" gorm:"column:bk_biz_id"`
+	PartitionColumn string `json:"partition_column" gorm:"column:partition_column"`
+	ImmuteDomain    string `json:"immute_domain" gorm:"column:immute_domain"`
+}
+
+func InitCustimazation() {
+	CustimazationMap = make(map[int64]string)
+	custimazations := []Custimazation{}
+	result := DB.Self.Session(&gorm.Session{
+		Logger: logger.Default.LogMode(logger.Info),
+	}).Table("partition_customization_config").Find(&custimazations)
+	if result.Error != nil {
+		slog.Error("定制化配置读取失败！", result.Error)
+	}
+	for _, cs := range custimazations {
+		if cs.ImmuteDomain != "" {
+			CustimazationMap[cs.BkBizId] = cs.ImmuteDomain
+		} else if cs.PartitionColumn != "" {
+			CustimazationMap[cs.BkBizId] = cs.PartitionColumn
+		}
+	}
+}

dbm-services/mysql/db-partition/service/check_partition_base_func.go

@@ -146,16 +146,9 @@ func (config *PartitionConfig) GetDbTableInfo(fromCron bool, host Host) (ptlist
 				slog.Error("GetDbTableInfo", sql, err.Error())
 				return nil, err
 			}
-			// (1)兼容【分区字段为空】的历史问题，对于某些特殊的分区类型，旧系统已经不在页面上支持，所以旧系统有意将分区字段留空，
-			// 		使其无法在页面编辑，避免改变了其他所属分区类别，因此无法核对比较，但不影响新增和删除分区。
-			// (2)兼容web、dnf业务的特殊定制类型，分区字段类型为int，但是系统记录为timestamp，因此无法核对比较，但不影响新增和删除分区。
-			// (3)兼容minigame业务的特殊定制类型，分区类型为0，但是实际定义与分区类型存在差异，因此无法核对比较，但不影响新增和删除分区。
-			webCustomization := config.BkBizId == 159 && config.PartitionColumn == "Fcreate_time"
-			iegamsCustomization := config.BkBizId == 5016839 && config.PartitionColumn == "Fcreate_time"
-			minigameCustomization := config.BkBizId == 121 && config.ImmuteDomain == "gamedb.game-record.minigame.db"
-			dnfCustomization := config.BkBizId == 105 && config.PartitionColumn == "occ_date"
-			if config.PartitionColumn != "" && !webCustomization && !iegamsCustomization &&
-				!minigameCustomization && !dnfCustomization {
+
+			flag := config.CustomizationCheck()
+			if flag {
 				// 分区表至少会有一个分区
 				for _, v := range output.CmdResults[0].TableData {
 					// 如果发现分区字段、分区间隔与规则不符合，需要重新做分区，页面调整了分区规则
@@ -215,6 +208,19 @@ func (config *PartitionConfig) GetDbTableInfo(fromCron bool, host Host) (ptlist
 	return ptlist, nil
 }
 
+func (config *PartitionConfig) CustomizationCheck() (flag bool) {
+	// (1)兼容【分区字段为空】的历史问题，对于某些特殊的分区类型，旧系统已经不在页面上支持，所以旧系统有意将分区字段留空，
+	// 		使其无法在页面编辑，避免改变了其他所属分区类别，因此无法核对比较，但不影响新增和删除分区。
+	// (2)兼容web、dnf业务的特殊定制类型，分区字段类型为int，但是系统记录为timestamp，因此无法核对比较，但不影响新增和删除分区。
+	// (3)兼容minigame业务的特殊定制类型，分区类型为0，但是实际定义与分区类型存在差异，因此无法核对比较，但不影响新增和删除分区。
+	var custFlag bool
+	if val, ok := model.CustimazationMap[config.BkBizId]; ok {
+		custFlag = config.PartitionColumn == val || config.ImmuteDomain == val
+	}
+	flag = config.PartitionColumn != "" && !custFlag
+	return flag
+}
+
 func CheckPartitionExpression(expression, method, column string, partitionType int) (bool, error) {
 	columnWithBackquote := fmt.Sprintf("`%s`", column)
 	switch partitionType {

dbm-services/mysql/db-partition/service/cron_basic_func.go

@@ -12,8 +12,6 @@ package service
 
 import (
 	"context"
-	"dbm-services/common/go-pubpkg/errno"
-	"dbm-services/mysql/db-partition/util"
 	"encoding/json"
 	"fmt"
 	"log/slog"
@@ -23,6 +21,9 @@ import (
 	"sync"
 	"time"
 
+	"dbm-services/common/go-pubpkg/errno"
+	"dbm-services/mysql/db-partition/util"
+
 	"golang.org/x/time/rate"
 
 	"dbm-services/mysql/db-partition/model"
@@ -357,7 +358,8 @@ func GetTendbclusterInstances(cluster string) (map[string][]SpiderNode, int, err
 	domain := tmp[0]
 	port, _ := strconv.Atoi(tmp[1])
 	cloud, _ := strconv.Atoi(tmp[2])
-	address := fmt.Sprintf("%s:%d", domain, port)
+	// port加1000，去任意中控查询集群信息
+	address := fmt.Sprintf("%s:%d", domain, port+1000)
 	var splitCnt int
 	var tdbctlPrimary string
 	// 查询tdbctl