Merge pull request #98 from nineaiyu/ndev

Ndev

Author: nineaiyu

common/apps.py

@@ -15,7 +15,6 @@ def ready(self):
         from .celery import heatbeat  # noqa
         from . import signal_handlers  # noqa
         from . import tasks  # noqa
-        from .swagger.utils import OpenApiAuthenticationScheme, OpenApiPrimaryKeyRelatedField  # noqa
         from .signals import django_ready
         excludes = ['migrate', 'compilemessages', 'makemigrations', 'stop']
         for i in excludes:

common/celery/logger.py

@@ -97,7 +97,8 @@ def flush(self):
             f.flush()
 
     def handle_task_start(self, task_id):
-        log_path = get_celery_task_log_path(task_id.split('_')[0])
+        # log_path = get_celery_task_log_path(task_id.split('_')[0])
+        log_path = get_celery_task_log_path(task_id)
         thread_id = self.get_current_thread_id()
         self.task_id_thread_id_mapper[task_id] = thread_id
         f = open(log_path, 'ab')

common/core/db/utils.py

@@ -107,27 +107,40 @@ def get_filter_attrs_qs(cls, rules):
         return filters
 
 
-def close_old_connections():
-    for conn in connections.all():
+def close_old_connections(**kwargs):
+    for conn in connections.all(initialized_only=True):
         conn.close_if_unusable_or_obsolete()
 
 
+# 这个要是在 Django 请求周期外使用的，不能影响 Django 的事务管理， 在 api 中使用会影响 api 事务
 @contextmanager
 def safe_db_connection():
-    in_atomic_block = connection.in_atomic_block  # 当前是否处于事务中
-    autocommit = transaction.get_autocommit()  # 是否启用了自动提交
-    created = False
+    close_old_connections()
+    yield
+    close_old_connections()
+
+
+@contextmanager
+def safe_atomic_db_connection(auto_close=False):
+    """
+    通用数据库连接管理器（线程安全、事务感知）：
+    - 在连接不可用时主动重建连接
+    - 在非事务环境下自动关闭连接（可选）
+    - 不影响 Django 请求/事务周期
+    """
+    in_atomic = connection.in_atomic_block  # 当前是否在事务中
+    autocommit = transaction.get_autocommit()
+    recreated = False
 
     try:
         if not connection.is_usable():
             connection.close()
             connection.connect()
-            created = True
+            recreated = True
         yield
     finally:
-        # 如果不是事务中（API 请求中可能需要提交事务），则关闭连接
-        if created and not in_atomic_block and autocommit:
-            print("close connection in safe_db_connection")
+        # 只在非事务、autocommit 模式下，才考虑主动清理连接
+        if auto_close or (recreated and not in_atomic and autocommit):
             close_old_connections()
 
 

common/core/fields.py

@@ -13,9 +13,7 @@
 from django.db.models.fields.files import FieldFile
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
-from rest_framework.fields import ChoiceField
 from rest_framework.request import Request
-from rest_framework.serializers import RelatedField, MultipleChoiceField
 
 from common.core.filter import get_filter_queryset
 from common.fields.utils import get_file_absolute_uri
@@ -33,22 +31,56 @@ def func(obj):
     return func(obj)
 
 
-class LabeledChoiceField(ChoiceField):
+class LabeledChoiceField(serializers.ChoiceField):
+    def __init__(self, **kwargs):
+        self.attrs = kwargs.pop("attrs", None) or ("value", "label")
+        super().__init__(**kwargs)
+
     def to_representation(self, key):
         if key is None:
             return key
         label = self.choices.get(key, key)
         return {"value": key, "label": label}
 
     def to_internal_value(self, data):
+        if not data:
+            return data
         if isinstance(data, dict):
             data = data.get("value")
         if isinstance(data, str) and "(" in data and data.endswith(")"):
             data = data.strip(")").split('(')[-1]
         return super(LabeledChoiceField, self).to_internal_value(data)
 
-
-class LabeledMultipleChoiceField(MultipleChoiceField):
+    def get_schema(self):
+        """
+        为 drf-spectacular 提供 OpenAPI schema
+        """
+        if getattr(self, 'many', False):
+            return {
+                'type': 'array',
+                'items': {
+                    'type': 'object',
+                    'properties': {
+                        'value': {'type': 'string'},
+                        'label': {'type': 'string'}
+                    }
+                },
+                'description': getattr(self, 'help_text', ''),
+                'title': getattr(self, 'label', ''),
+            }
+        else:
+            return {
+                'type': 'object',
+                'properties': {
+                    'value': {'type': 'string'},
+                    'label': {'type': 'string'}
+                },
+                'description': getattr(self, 'help_text', ''),
+                'title': getattr(self, 'label', ''),
+            }
+
+
+class LabeledMultipleChoiceField(serializers.MultipleChoiceField):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.choice_mapper = {
@@ -73,7 +105,7 @@ def to_internal_value(self, data):
             return data
 
 
-class BasePrimaryKeyRelatedField(RelatedField):
+class BasePrimaryKeyRelatedField(serializers.RelatedField):
     """
     Base class for primary key related fields.
     """
@@ -89,7 +121,7 @@ def __init__(self, attrs=None, ignore_field_permission=False, **kwargs):
         :param attrs: 默认为 None，返回默认的 pk， 一般需要自定义
         :param ignore_field_permission: 忽略字段权限控制
         """
-        self.attrs = attrs
+        self.attrs = attrs if attrs else ["pk"]
         self.label_format = kwargs.pop("format", None)
         self.input_type = kwargs.pop("input_type", None)
         self.input_type_prefix = kwargs.pop("input_type_prefix", None)
@@ -221,6 +253,122 @@ def to_internal_value(self, data):
         except (TypeError, ValueError):
             self.fail("incorrect_type", data_type=type(pk).__name__)
 
+    def get_schema(self):
+        """
+        为 drf-spectacular 提供 OpenAPI schema
+        """
+        # 获取字段的基本信息
+        field_type = 'array' if self.many else 'object'
+
+        if field_type == 'array':
+            # 如果是多对多关系
+            return {
+                'type': 'array',
+                'items': self._get_openapi_item_schema(),
+                'description': getattr(self, 'help_text', ''),
+                'title': getattr(self, 'label', ''),
+            }
+        else:
+            # 如果是一对一关系
+            return {
+                'type': 'object',
+                'properties': self._get_openapi_properties_schema(),
+                'description': getattr(self, 'help_text', ''),
+                'title': getattr(self, 'label', ''),
+            }
+
+    def _get_openapi_item_schema(self):
+        """
+        获取数组项的 OpenAPI schema
+        """
+        return self._get_openapi_object_schema()
+
+    def _get_openapi_object_schema(self):
+        """
+        获取对象的 OpenAPI schema
+        """
+        properties = {}
+
+        # 动态分析 attrs 中的属性类型
+        for attr in self.attrs:
+            # 尝试从 queryset 的 model 中获取字段信息
+            field_type = self._infer_field_type(attr)
+            properties[attr] = {
+                'type': field_type,
+                'description': f'{attr} field'
+            }
+
+        return {
+            'type': 'object',
+            'properties': properties,
+            'required': ['id'] if 'id' in self.attrs else []
+        }
+
+    def _infer_field_type(self, attr_name):
+        """
+        智能推断字段类型
+        """
+        try:
+            # 如果有 queryset，尝试从 model 中获取字段信息
+            if hasattr(self, 'queryset') and self.queryset is not None:
+                model = self.queryset.model
+                if hasattr(model, '_meta') and hasattr(model._meta, 'fields'):
+                    field = model._meta.get_field(attr_name)
+                    if field:
+                        return self._map_django_field_type(field)
+        except Exception:
+            pass
+
+        # 如果没有 queryset 或无法获取字段信息，使用启发式规则
+        return self._heuristic_field_type(attr_name)
+
+    def _map_django_field_type(self, field):
+        """
+        将 Django 字段类型映射到 OpenAPI 类型
+        """
+        field_type = type(field).__name__
+
+        # 整数类型
+        if 'Integer' in field_type or 'BigInteger' in field_type or 'SmallInteger' in field_type:
+            return 'integer'
+        # 浮点数类型
+        elif 'Float' in field_type or 'Decimal' in field_type:
+            return 'number'
+        # 布尔类型
+        elif 'Boolean' in field_type:
+            return 'boolean'
+        # 日期时间类型
+        elif 'DateTime' in field_type or 'Date' in field_type or 'Time' in field_type:
+            return 'string'
+        # 文件类型
+        elif 'File' in field_type or 'Image' in field_type:
+            return 'string'
+        # 其他类型默认为字符串
+        else:
+            return 'string'
+
+    def _heuristic_field_type(self, attr_name):
+        """
+        启发式推断字段类型
+        """
+        # 基于属性名的启发式规则
+
+        if attr_name in ['is_active', 'enabled', 'visible'] or attr_name.startswith('is_'):
+            return 'boolean'
+        elif attr_name in ['count', 'number', 'size', 'amount']:
+            return 'integer'
+        elif attr_name in ['price', 'rate', 'percentage']:
+            return 'number'
+        else:
+            # 默认返回字符串类型
+            return 'string'
+
+    def _get_openapi_properties_schema(self):
+        """
+        获取对象属性的 OpenAPI schema
+        """
+        return self._get_openapi_object_schema()['properties']
+
 
 class PhoneField(serializers.CharField):
 

common/core/serializers.py

@@ -28,6 +28,13 @@ class Meta:
         table_fields = []  # 用于控制前端table的字段展示
         tabs = []
 
+    def get_field_names(self, declared_fields, info):
+        """将默认的id字段 转换为 pk"""
+        fields = super().get_field_names(declared_fields, info)
+        if 'id' in fields:
+            return ['pk'] + [f for f in fields if f != 'id']
+        return fields
+
     def get_value(self, dictionary):
         # We override the default field access in order to support
         # nested HTML forms.

common/drf/parsers/csv.py

@@ -1,18 +1,18 @@
 # ~*~ coding: utf-8 ~*~
 #
+from functools import cached_property
 
 import chardet
 import unicodecsv
 
 from .base import BaseFileParser
 from ..const import CSV_FILE_ESCAPE_CHARS
-from ..utils import lazyproperty
 
 
 class CSVFileParser(BaseFileParser):
     media_type = 'text/csv'
 
-    @lazyproperty
+    @cached_property
     def match_escape_chars(self):
         chars = []
         for c in CSV_FILE_ESCAPE_CHARS:

common/drf/utils.py

@@ -94,16 +94,16 @@ def compile_i18n_file():
 
 
 def download_ip_db(force=False):
-    db_base_dir = os.path.join(APPS_DIR, 'common', 'utils', 'ip')
     db_path_url_mapper = {
-        ('geoip', 'GeoLite2-City.mmdb'): 'https://jms-pkg.oss-cn-beijing.aliyuncs.com/ip/GeoLite2-City.mmdb',
-        ('ipip', 'ipipfree.ipdb'): 'https://jms-pkg.oss-cn-beijing.aliyuncs.com/ip/ipipfree.ipdb'
+        ('system', 'GeoLite2-City.mmdb'): 'https://jms-pkg.oss-cn-beijing.aliyuncs.com/ip/GeoLite2-City.mmdb',
+        ('system', 'ipipfree.ipdb'): 'https://jms-pkg.oss-cn-beijing.aliyuncs.com/ip/ipipfree.ipdb'
     }
     for p, src in db_path_url_mapper.items():
-        path = os.path.join(db_base_dir, *p)
+        path = os.path.join(settings.DATA_DIR, *p)
         if not force and os.path.isfile(path) and os.path.getsize(path) > 1000:
             continue
         logger.info("Download ip db: {}".format(path))
+        os.makedirs(os.path.dirname(path), exist_ok=True)
         download_file(src, path)
 
 

common/management/commands/services/hands.py

@@ -23,6 +23,7 @@
 from common.celery.decorator import get_after_app_ready_tasks, get_after_app_shutdown_clean_tasks
 from common.celery.logger import CeleryThreadTaskFileHandler
 from common.celery.utils import get_celery_task_log_path
+from common.signals import django_ready
 from common.utils import get_logger
 from server.utils import get_current_request
 
@@ -162,3 +163,8 @@ def on_update_set_modifier(sender, instance=None, **kwargs):
 
 if settings.DEBUG_DEV:
     request_finished.connect(on_request_finished_logging_db_query)
+
+
+@receiver(django_ready)
+def clear_response_cache(sender, **kwargs):
+    cache.delete_pattern('magic_cache_response_*')