Support DeepSeekV3.2 with MLAPO operator

Signed-off-by: ZYang6263 <[email protected]>

[Feat]enable sfa cp for dsv3.2 (#4702)

RFC: vllm-project/vllm#30055

1. enable flashcommon1
export VLLM_ASCEND_ENABLE_FLASHCOMM1=1
2. enable sfa-cp
--additional-config '{ "enable_sfa_cp": true }' \

- vLLM version: v0.12.0
- vLLM main:
vllm-project/vllm@ad32e3e

Signed-off-by: AlvisGong <[email protected]>
Co-authored-by: clrs97 <[email protected]>
Co-authored-by: zzhx1 <[email protected]>
Co-authored-by: hwhaokun <[email protected]>
Co-authored-by: wangxiyuan <[email protected]>

Author: 5 people

vllm_ascend/ascend_config.py

@@ -80,6 +80,7 @@ def __init__(self, vllm_config):
                              enable_shared_expert_dp=True)
         self.multistream_overlap_shared_expert = additional_config.get(
             "multistream_overlap_shared_expert", False)
+        self.enable_sfa_cp = additional_config.get("enable_sfa_cp", False)
         self.recompute_scheduler_enable = additional_config.get(
             "recompute_scheduler_enable", False)
         self.lmhead_tensor_parallel_size = additional_config.get(

vllm_ascend/attention/sfa_v1.py

@@ -6,17 +6,21 @@
 from torch import nn
 from vllm.attention.backends.abstract import AttentionBackend, MLAAttentionImpl
 from vllm.config import VllmConfig, get_current_vllm_config
-from vllm.distributed import get_tensor_model_parallel_world_size, get_tp_group
 from vllm.model_executor.layers.linear import (LinearBase, ReplicatedLinear,
                                                UnquantizedLinearMethod)
 from vllm.triton_utils import HAS_TRITON
 from vllm.v1.attention.backends.utils import AttentionCGSupport
+from vllm.logger import logger
 
+from vllm_ascend import envs
 from vllm_ascend.ascend_config import get_ascend_config
 from vllm_ascend.attention.attention_v1 import AscendAttentionState
 from vllm_ascend.attention.mla_v1 import MAX_O_PROJ_PREFETCH_SIZE
 from vllm_ascend.attention.utils import (AscendCommonAttentionMetadata,
-                                         wait_for_kv_layer_from_connector)
+                                         wait_for_kv_layer_from_connector,
+                                         trans_rope_weight, transdata)
+from vllm_ascend.distributed.sfa_sp_context import (get_sfa_sp_context,
+                                                    set_sfa_sp_context)
 from vllm_ascend.ops.shared_weight_layer import (
     is_hidden_layer, post_process_after_loading_for_shared_weight_series,
     reach_layer_for_shared_weight_series,
@@ -26,6 +30,7 @@
 from vllm_ascend.utils import (ACL_FORMAT_FRACTAL_ND, ACL_FORMAT_FRACTAL_NZ,
                                _round_up, dispose_layer, enable_sp,
                                is_enable_nz, replace_layer)
+from vllm_ascend.quantization.w8a8 import AscendW8A8LinearMethod
 from vllm_ascend.worker.npu_input_batch import InputBatch
 
 if TYPE_CHECKING:
@@ -341,17 +346,54 @@ def __init__(
         self.enable_shared_expert_dp = ascend_config.enable_shared_expert_dp
         self.enable_prefetch = ascend_config.weight_prefetch_config.enabled
         self.enable_kv_nz = ascend_config.torchair_graph_config.enable_kv_nz
-        self.vllm_config = get_current_vllm_config()
+        self.enable_mlapo = envs.VLLM_ASCEND_ENABLE_MLAPO
+
         assert self.indexer is not None, "Indexer is required for DSA."
 
         self.enable_sfa_cp = enable_sp()
         self.local_num_heads = self.num_heads
-
+        self.vllm_config = get_current_vllm_config()
         if self.enable_sfa_cp:
             self.local_num_heads = self.num_heads * self.tp_size
 
-            #TODO: Temporarily adapt sfa-cp, remove after adapting near PCP. --clrs97
-            self._replace_linear_class_for_sfa_cp()
+            # Dispose tensor from the original q_proj
+            dispose_layer(self.q_proj)
+            # Construct the new q_proj using ReplicatedLinear
+            new_q_proj = ReplicatedLinear(
+                self.q_lora_rank,
+                self.local_num_heads * self.qk_head_dim,
+                bias=False,
+                quant_config=self.vllm_config.quant_config,
+                prefix=self.q_proj.prefix)
+            # Replace the q_proj with the new one
+            replace_layer(self.q_proj, new_q_proj)
+
+            # Dispose tensor from the original kv_b_proj
+            dispose_layer(self.kv_b_proj)
+            # Construct the new kv_b_proj using ReplicatedLinear
+            new_kv_b_proj = ReplicatedLinear(
+                self.kv_lora_rank,
+                self.local_num_heads *
+                (self.qk_nope_head_dim + self.v_head_dim),
+                bias=False,
+                quant_config=self.vllm_config.quant_config,
+                prefix=self.kv_b_proj.prefix)
+            # Replace the kv_b_proj with the new one
+            replace_layer(self.kv_b_proj, new_kv_b_proj)
+
+            # Dispose tensor from the original o_proj
+            dispose_layer(self.o_proj)
+            # Construct the new o_proj using ReplicatedLinear
+            config = self.vllm_config.model_config.hf_config
+            new_o_proj = ReplicatedLinear(
+                config.num_attention_heads * config.v_head_dim,
+                config.hidden_size,
+                bias=False,
+                quant_config=self.vllm_config.quant_config,
+                prefix=self.o_proj.prefix)
+            # Replace the o_proj with the new one
+            replace_layer(self.o_proj, new_o_proj)
+
             from vllm_ascend.distributed.parallel_state import \
                 get_shared_weight_group
             if is_hidden_layer(self.vllm_config, self.q_proj):
@@ -555,6 +597,98 @@ def rope_single(
         x = torch_npu.npu_interleave_rope(x, cos, sin)
         return x.view(B, N, D)
 
+    def _process_weights_for_fused_mlapo(self, act_dtype: torch.dtype):
+        assert self.kv_a_proj_with_mqa is None
+        assert self.fused_qkv_a_proj is not None
+
+        kv_a_proj_wt = self.fused_qkv_a_proj.weight.data[
+            ..., self.q_lora_rank:].contiguous()
+        q_a_proj_wt = self.fused_qkv_a_proj.weight.data[
+            ..., :self.q_lora_rank].contiguous()
+
+        self.fused_qkv_a_proj.weight = None
+
+        kv_a_proj_wt = kv_a_proj_wt.t().contiguous()
+        kv_a_proj_wt = trans_rope_weight(kv_a_proj_wt, self.qk_rope_head_dim)
+        kv_a_proj_wt = kv_a_proj_wt.t().contiguous()
+        wd_qkv = torch.cat((kv_a_proj_wt, q_a_proj_wt), dim=-1)
+        wd_qkv = wd_qkv.t().contiguous()
+        wd_qkv = transdata(wd_qkv,
+                           block_size=(16, 32)).unsqueeze(0).contiguous()
+        self.wd_qkv = torch_npu.npu_format_cast(wd_qkv, 29)
+
+        kv_a_proj_deq_scl = self.fused_qkv_a_proj.deq_scale[
+            self.q_lora_rank:].contiguous()
+        q_a_proj_deq_scl = self.fused_qkv_a_proj.deq_scale[:self.
+                                                           q_lora_rank].contiguous(
+                                                           )
+        kv_a_proj_deq_scl = kv_a_proj_deq_scl.reshape(
+            self.kv_lora_rank + self.qk_rope_head_dim, -1).contiguous()
+        kv_a_proj_deq_scl = trans_rope_weight(kv_a_proj_deq_scl,
+                                              self.qk_rope_head_dim)
+        kv_a_proj_deq_scl = kv_a_proj_deq_scl.view(
+            self.kv_lora_rank + self.qk_rope_head_dim).contiguous()
+        self.deq_scale_qkv = torch.cat((kv_a_proj_deq_scl, q_a_proj_deq_scl),
+                                       dim=-1).contiguous()
+
+        kv_a_proj_qt_bias = self.fused_qkv_a_proj.quant_bias[
+            self.q_lora_rank:].contiguous()
+        q_a_proj_qt_bias = self.fused_qkv_a_proj.quant_bias[:self.
+                                                            q_lora_rank].contiguous(
+                                                            )
+
+        kv_a_proj_qt_bias = kv_a_proj_qt_bias.reshape(
+            self.kv_lora_rank + self.qk_rope_head_dim, -1).contiguous()
+        kv_a_proj_qt_bias = trans_rope_weight(kv_a_proj_qt_bias,
+                                              self.qk_rope_head_dim)
+        kv_a_proj_qt_bias = kv_a_proj_qt_bias.view(
+            self.kv_lora_rank + self.qk_rope_head_dim).contiguous()
+        self.quant_bias_qkv = torch.cat((kv_a_proj_qt_bias, q_a_proj_qt_bias),
+                                        dim=-1).contiguous()
+
+        wu_q = self.q_proj.weight.data
+        wu_q = wu_q.t().reshape(self.num_heads,
+                                self.qk_nope_head_dim + self.qk_rope_head_dim,
+                                -1)
+        wu_q = trans_rope_weight(wu_q, self.qk_rope_head_dim)
+        wu_q = wu_q.reshape(
+            self.num_heads * (self.qk_nope_head_dim + self.qk_rope_head_dim),
+            -1)
+        wu_q = transdata(wu_q, block_size=(16, 32)).unsqueeze(0).contiguous()
+        self.wu_q = torch_npu.npu_format_cast(wu_q, 29)
+
+        qb_deq_scl = self.q_proj.deq_scale.data
+        qb_deq_scl = qb_deq_scl.reshape(
+            self.num_heads, self.qk_nope_head_dim + self.qk_rope_head_dim, -1)
+        qb_deq_scl = trans_rope_weight(qb_deq_scl, self.qk_rope_head_dim)
+        self.qb_deq_scl = qb_deq_scl.reshape(
+            self.num_heads * (self.qk_nope_head_dim + self.qk_rope_head_dim))
+
+        qb_qt_bias = self.q_proj.quant_bias.data
+        qb_qt_bias = qb_qt_bias.reshape(
+            self.num_heads, self.qk_nope_head_dim + self.qk_rope_head_dim, -1)
+        qb_qt_bias = trans_rope_weight(qb_qt_bias, self.qk_rope_head_dim)
+        self.qb_qt_bias = qb_qt_bias.reshape(
+            self.num_heads * (self.qk_nope_head_dim + self.qk_rope_head_dim))
+
+        device = self.q_proj.weight.device
+        self.gamma1 = self.q_a_layernorm.weight.data
+        self.beta1 = self.q_a_layernorm.bias.data
+        self.gamma2 = self.kv_a_layernorm.weight.data
+        self.quant_scale0 = self.fused_qkv_a_proj.input_scale.data
+        self.quant_offset0 = self.fused_qkv_a_proj.input_offset.data
+        self.quant_scale1 = self.q_proj.input_scale.data
+        self.quant_offset1 = self.q_proj.input_offset.data
+        self.ctkv_scale = torch.tensor([1], dtype=act_dtype, device=device)
+        self.q_nope_scale = torch.tensor([1], dtype=act_dtype, device=device)
+
+        if self.vllm_config.kv_transfer_config is not None:
+            self.fused_qkv_a_proj.deq_scale = None
+            self.fused_qkv_a_proj.quant_bias = None
+            self.q_proj.deq_scale = None
+            self.q_proj.quant_bias = None
+            torch.npu.empty_cache()
+
     def forward(
         self,
         layer_name,
@@ -578,56 +712,116 @@ def forward(
             return output.fill_(0)
         has_prefill = attn_metadata.has_prefill
         num_actual_tokens = attn_metadata.num_actual_tokens
+        cos = attn_metadata.cos
+        sin = attn_metadata.sin
+        cum_query_lens = attn_metadata.cum_query_lens
+        seq_lens = attn_metadata.seq_lens
+        actual_seq_lengths_query = cum_query_lens
+        actual_seq_lengths_key = seq_lens
         hidden_states = hidden_states[:num_actual_tokens]
         if self.enable_sfa_cp:
             need_gather_q_kv = False
         # Inputs and outputs may be padded for CUDA graphs
         output_padded = output
         output = output[:num_actual_tokens]
-        assert self.fused_qkv_a_proj is not None, "q lora is required for DSA."
-        maybe_npu_prefetch(inputs=self.fused_qkv_a_proj.weight,
-                           dependency=hidden_states,
-                           enabled=self.enable_prefetch)
-        qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
-        q_c, kv_no_split = qkv_lora.split(
-            [self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
-            dim=-1,
-        )
-        q_c = self.q_a_layernorm(q_c)
-
-        # Process for Flash Comm V1
-        if need_gather_q_kv:
-            q_c = torch.ops.vllm.maybe_all_gather_and_maybe_unpad(
-                q_c.contiguous(), need_gather_q_kv)
-            kv_no_split = torch.ops.vllm.maybe_all_gather_and_maybe_unpad(
-                kv_no_split.contiguous(), need_gather_q_kv)
-
-        if has_prefill:
-            wait_for_kv_layer_from_connector(layer_name)
 
-        cos = attn_metadata.cos
-        sin = attn_metadata.sin
-        slot_mapping = attn_metadata.slot_mapping[:num_actual_tokens]
-        slot_mapping_cp = None
-        actual_seq_lengths_query = attn_metadata.cum_query_lens
-        actual_seq_lengths_key = attn_metadata.seq_lens
-        if self.enable_sfa_cp:
-            assert attn_metadata.sfa_cp_context is not None
-            slot_mapping_cp = attn_metadata.sfa_cp_context.slot_mapping_cp
-            actual_seq_lengths_query = attn_metadata.sfa_cp_context.actual_seq_lengths_query
-            actual_seq_lengths_key = attn_metadata.sfa_cp_context.actual_seq_lengths_key
-
-        self.exec_kv(kv_no_split, cos, sin, kv_cache, slot_mapping,
-                     slot_mapping_cp)
-
-        if self.enable_sfa_cp and attn_metadata.sfa_cp_context is not None:
-            if is_hidden_layer(self.vllm_config, self.q_proj):
-                reach_layer_for_shared_weight_series(self.q_proj)
-            if is_hidden_layer(self.vllm_config, self.o_proj):
-                reach_layer_for_shared_weight_series(self.o_proj)
-
-        ql_nope, q_pe = self._q_proj_and_k_up_proj(q_c)
-        q_pe = self.rope_single(q_pe, cos, sin)
+        from vllm.forward_context import get_forward_context
+        forward_context = get_forward_context()
+        if self.enable_mlapo and not forward_context.with_prefill:
+            hidden_states = torch.ops.vllm.maybe_all_gather_and_maybe_unpad(
+                hidden_states.contiguous(), need_gather_q_kv)
+            k_nope, k_pe = kv_cache[0], kv_cache[1]
+            ql_nope = torch.empty(
+                (num_actual_tokens, self.W_UK_T.shape[0], k_nope.shape[-1]),
+                dtype=hidden_states.dtype,
+                device=hidden_states.device,
+            )
+            q_pe = torch.empty(
+                (num_actual_tokens, self.W_UK_T.shape[0], k_pe.shape[-1]),
+                dtype=hidden_states.dtype,
+                device=hidden_states.device,
+            )
+            q_c = torch.empty(
+                (num_actual_tokens, self.q_lora_rank),
+                dtype=hidden_states.dtype,
+                device=hidden_states.device,
+            )
+            torch.ops._C_ascend.mla_preprocess(
+                hidden_states,
+                self.wd_qkv,
+                self.deq_scale_qkv,
+                self.gamma1,
+                self.beta1,
+                self.wu_q,
+                self.qb_deq_scl,
+                self.gamma2,
+                cos,
+                sin,
+                self.W_UK_T,
+                k_nope,
+                k_pe,
+                attn_metadata.slot_mapping[:num_actual_tokens].flatten(),
+                quant_scale0=self.quant_scale0,
+                quant_offset0=self.quant_offset0,
+                bias0=self.quant_bias_qkv,
+                quant_scale1=self.quant_scale1,
+                quant_offset1=self.quant_offset1,
+                bias1=self.qb_qt_bias,
+                ctkv_scale=self.ctkv_scale,
+                q_nope_scale=self.q_nope_scale,
+                cache_mode="krope_ctkv",
+                quant_mode="per_tensor_quant_asymm",
+                enable_inner_out=True,
+                q_out0=ql_nope,
+                kv_cache_out0=k_nope,
+                q_out1=q_pe,
+                kv_cache_out1=k_pe,
+                inner_out=q_c,
+            )
+        else:
+            assert self.fused_qkv_a_proj is not None, "q lora is required for DSA."
+            maybe_npu_prefetch(inputs=self.fused_qkv_a_proj.weight,
+                            dependency=hidden_states,
+                            enabled=self.enable_prefetch)
+            qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
+            q_c, kv_no_split = qkv_lora.split(
+                [self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
+                dim=-1,
+            )
+            q_c = self.q_a_layernorm(q_c)
+            # Process for Flash Comm V1
+            if need_gather_q_kv:
+                q_c = torch.ops.vllm.maybe_all_gather_and_maybe_unpad(
+                        q_c.contiguous(), need_gather_q_kv)
+                kv_no_split = torch.ops.vllm.maybe_all_gather_and_maybe_unpad(
+                    kv_no_split.contiguous(), need_gather_q_kv)
+
+            if has_prefill:
+                wait_for_kv_layer_from_connector(layer_name)
+
+            cos = attn_metadata.cos
+            sin = attn_metadata.sin
+            slot_mapping = attn_metadata.slot_mapping[:num_actual_tokens]
+            slot_mapping_cp = None
+            actual_seq_lengths_query = attn_metadata.cum_query_lens
+            actual_seq_lengths_key = attn_metadata.seq_lens
+            if self.enable_sfa_cp:
+                assert attn_metadata.sfa_cp_context is not None
+                slot_mapping_cp = attn_metadata.sfa_cp_context.slot_mapping_cp
+                actual_seq_lengths_query = attn_metadata.sfa_cp_context.actual_seq_lengths_query
+                actual_seq_lengths_key = attn_metadata.sfa_cp_context.actual_seq_lengths_key
+
+            self.exec_kv(kv_no_split, cos, sin, kv_cache, slot_mapping,
+                        slot_mapping_cp)
+
+            if self.enable_sfa_cp and attn_metadata.sfa_cp_context is not None:
+                if is_hidden_layer(self.vllm_config, self.q_proj):
+                    reach_layer_for_shared_weight_series(self.q_proj)
+                if is_hidden_layer(self.vllm_config, self.o_proj):
+                    reach_layer_for_shared_weight_series(self.o_proj)
+
+            ql_nope, q_pe = self._q_proj_and_k_up_proj(q_c)
+            q_pe = self.rope_single(q_pe, cos, sin)
 
         topk_indices = self.indexer_select(
             x=hidden_states,