feat(backend): Optimize the way of collecting data from multiple database instances using probes. issue: #14123

Author: ylgeeker

dbm-services/common/dbha-v2/etc/probe.yaml

@@ -10,12 +10,25 @@ reporter:
     connTimeout: 5s
 
 harvester:
-  - name: mysql
-    endpoint: ""
-    user: root
-    password: 123456
+  mysql:
+    user: "root"
+    password: "root"
+    # The time interval for each data collection from MySQL.
     interval: 20s
 
+    endpoints:
+      # If this 'proto' is not set, the default value is 'tcp'.
+      - proto: tcp
+        clusterType: ""
+        machineType: ""
+        accessLayer: ""
+        ip: 127.0.0.1
+
+        # Ports configuration format is: 1000~2000,20003, 3000, 5000-6000
+        # The target port number to be probed are such that begin <= port <= end.
+        ports: [1000-2000,20003, 3000, 5000-6000]
+        adminPorts: []
+
 log:
   path: ./logs/probe.log
   level: debug

dbm-services/common/dbha-v2/etc/probe_harvester.yaml

@@ -0,0 +1,5 @@
+dbType: mysql
+endpoint: ""
+interval: 20s
+user: root
+password: 123456

dbm-services/common/dbha-v2/go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/viper v1.20.1
-	github.com/stretchr/testify v1.11.1
 	github.com/xdg-go/scram v1.1.2
 	go.etcd.io/etcd/client/v3 v3.6.0
 	go.uber.org/zap v1.27.0
@@ -72,7 +71,6 @@ require (
 require (
 	github.com/ebitengine/purego v0.8.4 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/shirou/gopsutil/v4 v4.25.6
 	github.com/tklauser/go-sysconf v0.3.12 // indirect

dbm-services/common/dbha-v2/internal/probe/config/config.go

@@ -41,21 +41,36 @@ var Cfg = Configuration{
 	},
 }
 
-// ReporterConfig reporter's config
+// ReporterConfig reporter config
 type ReporterConfig struct {
 	Name        string        `yaml:"name"        mapstructure:"name"`
 	Endpoint    string        `yaml:"endpoint"    mapstructure:"endpoint"`
 	DataID      uint64        `yaml:"dataID"      mapstructure:"dataID"`
 	ConnTimeout time.Duration `yaml:"connTimeout" mapstructure:"connTimeout"`
 }
 
-// HarvesterConfig harvester's config
+// DbEndpointConfig db instance endpoint config
+type DbEndpointConfig struct {
+	Proto       string   `yaml:"proto"       mapstructure:"proto"`
+	ClusterType string   `yaml:"clusterType" mapstructure:"clusterType"`
+	MachineType string   `yaml:"machineType" mapstructure:"machineType"`
+	AccessLayer string   `yaml:"accessLayer" mapstructure:"accessLayer"`
+	Ip          string   `yaml:"ip"          mapstructure:"ip"`
+	Ports       []string `yaml:"ports"       mapstructure:"ports"`
+	AdminPorts  []string `yaml:"adminPorts"  mapstructure:"adminPorts"`
+}
+
+// MySqlHarvesterConfig MySQL harvester config
+type MySqlHarvesterConfig struct {
+	User      string             `yaml:"user"      mapstructure:"user"`
+	Password  string             `yaml:"password"  mapstructure:"password"`
+	Interval  time.Duration      `yaml:"interval"  mapstructure:"interval"`
+	Endpoints []DbEndpointConfig `yaml:"endpoints" mapstructure:"endpoints"`
+}
+
+// HarvesterConfig harvester config
 type HarvesterConfig struct {
-	Name     string        `yaml:"name"     mapstructure:"name"`
-	Endpoint string        `yaml:"endpoint" mapstructure:"endpoint"`
-	User     string        `yaml:"user"     mapstructure:"user"`
-	Password string        `yaml:"password" mapstructure:"password"`
-	Interval time.Duration `yaml:"interval" mapstructure:"interval"`
+	MySql *MySqlHarvesterConfig
 }
 
 // LogConfig log configuration
@@ -68,11 +83,11 @@ type LogConfig struct {
 
 // Configuration receiver's configuration
 type Configuration struct {
-	Name       string            `yaml:"name"      mapstructure:"name"`
-	Version    string            `yaml:"version"   mapstructure:"version"`
-	ServiceID  string            `yaml:"serviceID" mapstructure:"serviceID"`
-	PidFile    string            `yaml:"pidFile"   mapstructure:"pidFile"`
-	Reporters  []ReporterConfig  `yaml:"reporter"  mapstructure:"reporter"`
-	Harvesters []HarvesterConfig `yaml:"harvester" mapstructure:"harvester"`
-	Log        LogConfig         `yaml:"log"       mapstructure:"log"`
+	Name      string           `yaml:"name"      mapstructure:"name"`
+	Version   string           `yaml:"version"   mapstructure:"version"`
+	ServiceID string           `yaml:"serviceID" mapstructure:"serviceID"`
+	PidFile   string           `yaml:"pidFile"   mapstructure:"pidFile"`
+	Reporters []ReporterConfig `yaml:"reporter"  mapstructure:"reporter"`
+	Harvester HarvesterConfig  `yaml:"harvester" mapstructure:"harvester"`
+	Log       LogConfig        `yaml:"log"       mapstructure:"log"`
 }

dbm-services/common/dbha-v2/internal/probe/harvester/harvester.go

@@ -25,27 +25,11 @@
 package harvester
 
 import (
-	"strings"
-
-	"dbm-services/common/dbha-v2/internal/probe/config"
 	"dbm-services/common/dbha-v2/internal/probe/harvester/mysql"
-	"dbm-services/common/dbha-v2/internal/probe/harvester/plugin"
-	"dbm-services/common/dbha-v2/pkg/gerrors"
-	"dbm-services/common/dbha-v2/pkg/logger"
 )
 
 var (
-	ErrUnknownPlugin = gerrors.Newf(gerrors.Unknown, "unknown harvester plugin")
+	// NewPluginMySql To avoid potential ambiguity caused by directly using 'mysql',
+	// the method for creating the mysql plugin has been renamed here.
+	NewPluginMySql = mysql.NewMySql
 )
-
-// NewPlugin creates a new probe plugin.
-func NewPlugin(cfg config.HarvesterConfig) (plugin.Plugin, error) {
-	switch strings.ToLower(cfg.Name) {
-	case strings.ToLower(mysql.Name):
-		return mysql.NewMySql(cfg)
-
-	default:
-		logger.Error("unknown harvester plugin: %s", cfg.Name)
-		return nil, ErrUnknownPlugin
-	}
-}

dbm-services/common/dbha-v2/internal/probe/harvester/harvester_test.go

@@ -26,29 +26,8 @@ package harvester_test
 
 import (
 	"testing"
-
-	"dbm-services/common/dbha-v2/internal/probe/config"
-	"dbm-services/common/dbha-v2/internal/probe/harvester"
-
-	"github.com/stretchr/testify/assert"
 )
 
 func TestNewPlugin(t *testing.T) {
-	tests := []struct {
-		name string
-		cfg  config.HarvesterConfig
-	}{
-		{"mysql", config.HarvesterConfig{Name: "mysql"}},
-		{"redis", config.HarvesterConfig{Name: "redis"}},
-		{"mysql uppercase", config.HarvesterConfig{Name: "MYSQL"}},
-	}
-
-	for _, tt := range tests {
 
-		t.Run(tt.name, func(t *testing.T) {
-			plugin, err := harvester.NewPlugin(tt.cfg)
-			assert.NoError(t, err)
-			assert.NotNil(t, plugin)
-		})
-	}
 }