fix(backend): 丰富filter_clusters过滤字段 #7495

Author: iSecloud

dbm-ui/backend/core/storages/base.py

@@ -36,9 +36,9 @@ def get_available_name(self, name, max_length=None):
         file_root, file_ext = os.path.splitext(file_name)
 
         def _gen_random_name(_file_root) -> str:
-            # 在文件名的起始位置添加随机串，源码规则为 "%s_%s%s" % (_file_root, get_random_string(7), file_ext)
+            # 在文件名的起始位置添加随机串，源码规则为 "%s_dbmrpt_%s%s" % (_file_root, get_random_string(7), file_ext)
             # 上述规则对 .tar.gz 不友好，会在类型后缀中间加随机串，所以改为随机串作为前缀
-            return os.path.join(dir_name, "%s_%s%s" % (get_random_string(7), _file_root, file_ext))
+            return os.path.join(dir_name, "%s_dbmrpt_%s%s" % (get_random_string(7), _file_root, file_ext))
 
         # not self.file_overwrite and self.exists(name) 利用 and 短路特点，如果 file_overwrite=True 就无需校验文件是否存在
         while (not self.file_overwrite and self.exists(name)) or (max_length and len(name) > max_length):

dbm-ui/backend/db_services/dbbase/resources/query.py

@@ -373,7 +373,9 @@ def _list_clusters(
         # 定义内置的过滤参数map
         filter_params_map = filter_params_map or {}
         inner_filter_params_map = {
+            # 集群id
             "id": Q(id=query_params.get("id")),
+            # 集群名/别名
             "name": (
                 Q(name__in=query_params.get("name", "").split(","))
                 | Q(alias__in=query_params.get("name", "").split(","))
@@ -384,7 +386,9 @@ def _list_clusters(
             "major_version": Q(major_version__in=query_params.get("major_version", "").split(",")),
             # 地域
             "region": Q(region__in=query_params.get("region", "").split(",")),
+            # 集群id列表
             "cluster_ids": Q(id__in=query_params.get("cluster_ids")),
+            # 创建者
             "creator": Q(creator__icontains=query_params.get("creator")),
             # 所属DB模块
             "db_module_id": Q(db_module_id__in=query_params.get("db_module_id", "").split(",")),
@@ -394,8 +398,8 @@ def _list_clusters(
             "status": Q(status__in=query_params.get("status", "").split(",")),
             # 时区
             "time_zone": Q(time_zone=query_params.get("time_zone", "").split(",")),
-            # 域名精确查询，主要用于工具箱手动填入域名查询
-            "exact_domain": Q(immute_domain=query_params.get("exact_domain")),
+            # 主域名精确查询，主要用于工具箱手动填入域名查询
+            "exact_domain": Q(immute_domain__in=query_params.get("exact_domain", "").split(",")),
             # 域名
             "domain": build_q_for_domain_by_cluster(domains=query_params.get("domain", "").split(",")),
         }

dbm-ui/backend/db_services/dbbase/resources/serializers.py

@@ -20,19 +20,18 @@
 class ListResourceSLZ(serializers.Serializer):
     id = serializers.IntegerField(required=False)
     name = serializers.CharField(required=False)
-    ip = serializers.CharField(required=False)
     instance = serializers.CharField(required=False)
     domain = serializers.CharField(required=False)
     creator = serializers.CharField(required=False)
     major_version = serializers.CharField(required=False)
     region = serializers.CharField(required=False)
     cluster_ids = serializers.ListField(child=serializers.IntegerField(), required=False, allow_empty=True)
     exact_domain = serializers.CharField(help_text=_("精确域名查询"), required=False)
-    ordering = serializers.CharField(required=False, help_text=_("排序字段,非必填"))
     status = serializers.CharField(required=False, help_text=_("状态"))
     db_module_id = serializers.CharField(required=False, help_text=_("所属DB模块"))
     bk_cloud_id = serializers.CharField(required=False, help_text=_("管控区域"))
     cluster_type = serializers.CharField(required=False, help_text=_("集群类型"))
+    ordering = serializers.CharField(required=False, help_text=_("排序字段,非必填"))
 
 
 class ListMySQLResourceSLZ(ListResourceSLZ):

dbm-ui/backend/db_services/dbbase/serializers.py

@@ -8,17 +8,16 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-import re
 
 from django.db.models import Q
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from backend.components import CCApi
-from backend.constants import IP_PORT_DIVIDER, IP_PORT_RE_PATTERN
 from backend.db_dirty.models import DirtyMachine
 from backend.db_meta.enums import ClusterPhase, ClusterType
 from backend.db_services.dbbase.constants import ResourceType
+from backend.db_services.dbbase.resources.serializers import ListResourceSLZ
 from backend.db_services.ipchooser.query.resource import ResourceQueryHelper
 from backend.db_services.redis.resources.redis_cluster.query import RedisListRetrieveResource
 from backend.dbm_init.constants import CC_APP_ABBR_ATTR
@@ -76,33 +75,23 @@ class Meta:
         swagger_schema_fields = {"example": []}
 
 
-class ClusterFilterSerializer(serializers.Serializer):
+class ClusterFilterSerializer(ListResourceSLZ):
+    # 基础的集群过滤条件
     bk_biz_id = serializers.IntegerField(help_text=_("业务ID"))
-    exact_domain = serializers.CharField(help_text=_("域名精确查询(逗号分割)"), required=False, default="")
     cluster_ids = serializers.CharField(help_text=_("集群ID(逗号分割)"), required=False, default="")
-
-    # 后续有其他过滤条件可以再加
-    cluster_type = serializers.CharField(help_text=_("集群类型"), required=False)
-    instance = serializers.CharField(help_text=_("实例查询(逗号分割)"), required=False, default="")
+    cluster_type = serializers.CharField(help_text=_("集群类型"), required=False, default="")
 
     def validate(self, attrs):
-        cluster_ids = attrs["cluster_ids"].split(",") if attrs["cluster_ids"] else []
-        exact_domains = attrs["exact_domain"].split(",") if attrs["exact_domain"] else []
-        instances = attrs["instance"].split(",") if attrs["instance"] else []
+        # 获取集群基础过滤条件用作第一轮过滤
         filters = Q(bk_biz_id=attrs["bk_biz_id"])
-        filters &= Q(id__in=cluster_ids) if cluster_ids else Q()
-        filters &= Q(immute_domain__in=exact_domains) if exact_domains else Q()
-        filters &= Q(cluster_type=attrs["cluster_type"]) if attrs.get("cluster_type") else Q()
-        instance_filters = Q()
-        for instance in instances:
-            if re.compile(IP_PORT_RE_PATTERN).match(instance):
-                ip, port = instance.split(IP_PORT_DIVIDER)
-                instance_filter = Q(storageinstance__machine__ip=ip, storageinstance__port=port) | Q(
-                    proxyinstance__machine__ip=ip, proxyinstance__port=port
-                )
-                instance_filters |= instance_filter
-        filters &= instance_filters
+        if attrs["cluster_ids"]:
+            filters &= Q(id__in=attrs["cluster_ids"].split(","))
+        if attrs["cluster_type"]:
+            filters &= Q(cluster_type__in=attrs["cluster_type"].split(","))
         attrs["filters"] = filters
+        # 补充list resource过滤条件
+        query_params = {field: attrs[field] for field in self.fields.keys() if field in attrs}
+        attrs["query_params"] = query_params
         return attrs
 
 

dbm-ui/backend/db_services/dbbase/views.py

@@ -117,16 +117,17 @@ def common_query_cluster(self, request, *args, **kwargs):
     @action(methods=["GET"], detail=False, serializer_class=ClusterFilterSerializer)
     def filter_clusters(self, request, *args, **kwargs):
         data = self.params_validate(self.get_serializer_class())
-        clusters = Cluster.objects.filter(data["filters"])
         # 先按照集群类型聚合
-        cluster_type__clusters: Dict[str, List[Cluster]] = defaultdict(list)
-        for cluster in clusters:
-            cluster_type__clusters[cluster.cluster_type].append(cluster)
+        resource_cls__cluster_ids_map = defaultdict(list)
+        for cluster in Cluster.objects.filter(data["filters"]).values("id", "cluster_type"):
+            resource_class = register.cluster_type__resource_class[cluster["cluster_type"]]
+            resource_cls__cluster_ids_map[resource_class].append(cluster["id"])
         # 按照不同的集群类型，调用不同的query resource去查询集群数据
         clusters_data: List[Dict] = []
-        for cluster_type, clusters in cluster_type__clusters.items():
-            query_params = {"cluster_ids": [cluster.id for cluster in clusters]}
-            resource_class: ListRetrieveResource = register.cluster_type__resource_class[cluster_type]
+        for resource_class, cluster_ids in resource_cls__cluster_ids_map.items():
+            if not list(cluster_ids):
+                continue
+            query_params = {**data["query_params"], "cluster_ids": list(cluster_ids)}
             cluster_resource_data: ResourceList = resource_class.list_clusters(
                 bk_biz_id=data["bk_biz_id"], query_params=query_params, limit=-1, offset=0
             )

dbm-ui/backend/db_services/mysql/sql_import/handlers.py

@@ -259,6 +259,7 @@ def _get_user_semantic_tasks(self, cluster_type, code) -> List[Dict]:
         flow_tree_list = FlowTree.objects.filter(root_id__in=task_ids)
         semantic_info_list = [
             {
+                "cluster_type": cluster_type,
                 "bk_biz_id": tree.bk_biz_id,
                 "root_id": tree.root_id,
                 "created_at": tree.created_at,

dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py

@@ -27,11 +27,13 @@
 from backend.db_meta.models.cluster import Cluster
 from backend.db_services.dbbase.resources import query
 from backend.db_services.dbbase.resources.query import ResourceList
+from backend.db_services.dbbase.resources.register import register_resource_decorator
 from backend.db_services.ipchooser.query.resource import ResourceQueryHelper
 from backend.db_services.redis.resources.constants import SQL_QUERY_MASTER_SLAVE_STATUS
 from backend.utils.basic import dictfetchall
 
 
+@register_resource_decorator()
 class RedisListRetrieveResource(query.ListRetrieveResource):
     """查看twemproxy-redis架构的资源"""
 
@@ -141,8 +143,8 @@ def _to_cluster_representation(
         if machine_list:
             spec_id = cluster.storages[0].machine.spec_id
             spec = redis_spec_map.get(spec_id)
-            cluster_spec = model_to_dict(spec)
-            cluster_capacity = spec.capacity * machine_pair_cnt
+            cluster_spec = model_to_dict(spec) if spec else {}
+            cluster_capacity = spec.capacity * machine_pair_cnt if spec else 0
 
         # dns是否指向clb
         dns_to_clb = cluster.clusterentry_set.filter(