Merge branch 'master' into fix_xattn_reference

Author: vshampor

src/bindings/python/tests/test_runtime/test_tensor.py

@@ -167,10 +167,12 @@ def get_tensor():
         param2 = ops.parameter(ov.Shape([1, 3, 32, 32]), dtype=np.float64)
         param3 = ops.parameter(ov.PartialShape.dynamic(), dtype=np.float64)
         ones_arr = np.ones(shape=(1, 3, 32, 32), dtype=np.float64)
-        assert sys.getrefcount(ones_arr) == 2
+        # Account for changes to refcount in Python 3.14+
+        expected_refcount_offset = 1 if sys.version_info >= (3, 14) else 0
+        assert sys.getrefcount(ones_arr) == 2 - expected_refcount_offset
         tensor1 = ov.Tensor(param1.output(0))
         tensor2 = ov.Tensor(param2.output(0), ones_arr)
-        assert sys.getrefcount(ones_arr) == 3
+        assert sys.getrefcount(ones_arr) == 3 - expected_refcount_offset
         tensor3 = ov.Tensor(param3.output(0))
         tensor4 = ov.Tensor(param3.output(0), ones_arr)
         assert tensor1.shape == param1.shape
@@ -195,11 +197,13 @@ def get_tensor():
         compiled_model = generate_relu_compiled_model(device)
         output = compiled_model.output(0)
         ones_arr = np.ones(shape=(1, 3, 32, 32), dtype=np.float32)
-        assert sys.getrefcount(ones_arr) == 2
+        # Account for changes to refcount in Python 3.14+
+        expected_refcount_offset = 1 if sys.version_info >= (3, 14) else 0
+        assert sys.getrefcount(ones_arr) == 2 - expected_refcount_offset
 
         tensor1 = ov.Tensor(output)
         tensor2 = ov.Tensor(output, ones_arr)
-        assert sys.getrefcount(ones_arr) == 3
+        assert sys.getrefcount(ones_arr) == 3 - expected_refcount_offset
 
         output_node = output.get_node()
         assert tensor1.shape == output_node.shape
@@ -607,11 +611,13 @@ def test_init_from_list(init_value):
 def test_tensor_keeps_memory():
     def get_tensor():
         arr = np.ones((8, 16, 300), dtype=np.float32)
-        assert sys.getrefcount(arr) == 2
+        # Account for changes to refcount in Python 3.14+
+        expected_refcount_offset = 1 if sys.version_info >= (3, 14) else 0
+        assert sys.getrefcount(arr) == 2 - expected_refcount_offset
 
         shared_tensor = ov.Tensor(arr, shared_memory=True)
         arr[0][0][0:2] = 0
-        assert sys.getrefcount(arr) == 3
+        assert sys.getrefcount(arr) == 3 - expected_refcount_offset
 
         del arr
         return shared_tensor

src/plugins/intel_gpu/src/graph/graph_optimizer/prepare_primitive_fusing.cpp

@@ -43,6 +43,7 @@
 #include "reduce_inst.h"
 #include "group_normalization_inst.h"
 #include "lora_inst.h"
+#include "broadcast_inst.h"
 #include <vector>
 #include <map>
 #include <list>
@@ -751,6 +752,23 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) {
             return lora_is_single_user && is_simple_lora;
         };
 
+        auto broadcast_supports_fusings = [&](broadcast_node& bcast_node) -> bool {
+            if (bcast_node.get_outputs_count() != 1)
+                return false;
+
+            bool out_eltw = bcast_node.get_users().front()->is_type<eltwise>();
+            if (!out_eltw)
+                return false;
+
+            auto input_layout = bcast_node.get_output_layout();
+            auto output_layout = bcast_node.get_users().front()->get_output_layout();
+            if (input_layout.data_type != output_layout.data_type) {
+                return false;
+            }
+
+            return true;
+        };
+
         auto fuse_activation_f = [&](activation_node& activation_node) {
             GPU_DEBUG_IF(p.get_config().get_disable_post_ops_fusions() != 0) {
                 GPU_DEBUG_IF(p.get_config().get_disable_post_ops_fusions() != 11)
@@ -1055,7 +1073,9 @@ void prepare_primitive_fusing::fuse_simple_primitives(program &p) {
                                        reduce_supports_fusings(parents[i].first->as<reduce>())) ||
                                       (parents[i].first->is_type<lrn>()) ||
                                       (parents[i].first->is_type<lora>() &&
-                                       lora_supports_fusings(parents[i].first->as<lora>()));
+                                       lora_supports_fusings(parents[i].first->as<lora>())) ||
+                                       (parents[i].first->is_type<broadcast>() &&
+                                       broadcast_supports_fusings(parents[i].first->as<broadcast>()));
             }
 
             // Disable fusion to a node on constant path when second input is in data flow

src/plugins/intel_gpu/src/graph/impls/ocl_v2/sdpa/sdpa_base.cpp

@@ -348,6 +348,21 @@ kernel_impl_params SDPABase::static_canonicalize_shapes(const kernel_impl_params
         return pshape;
     };
 
+    auto extend_padding_to_rank_in_num_heads_dim = [](const padding& input_padding, size_t input_rank, size_t target_rank = 4) {
+        if (input_rank == target_rank) {
+            return input_padding;
+        }
+
+        std::vector<ov::Dimension::value_type> pad_low(input_padding._lower_size.begin(), input_padding._lower_size.begin() + input_rank);
+        std::vector<ov::Dimension::value_type> pad_up(input_padding._upper_size.begin(), input_padding._upper_size.begin() + input_rank);
+
+        const size_t num_heads_dim = 1;
+        pad_low.insert(pad_low.begin() + num_heads_dim, 0);
+        pad_up.insert(pad_up.begin() + num_heads_dim, 0);
+
+        return padding(pad_low, pad_up, input_padding._dynamic_dims_mask);
+    };
+
     const auto attn_mask_idx = 3;
     if (updated_impl_params.input_layouts.size() > attn_mask_idx) {
         const auto attn_mask_shape = updated_impl_params.input_layouts[attn_mask_idx].get_partial_shape();
@@ -356,8 +371,12 @@ kernel_impl_params SDPABase::static_canonicalize_shapes(const kernel_impl_params
 
     // For scale of 1D tensor or attention_mask of empty shape, use extend_shape_to_rank_from_end as before
     for (auto& input_layout : updated_impl_params.input_layouts) {
-        input_layout.set_partial_shape(input_layout.get_partial_shape().size() <= 1 ? extend_shape_to_rank_from_end(input_layout.get_partial_shape())
-                                                                                    : extend_pshape_to_rank_in_num_heads_dim(input_layout.get_partial_shape()));
+        size_t input_rank = input_layout.get_partial_shape().size();
+        input_layout.set_partial_shape(input_rank <= 1 ? extend_shape_to_rank_from_end(input_layout.get_partial_shape())
+                                                       : extend_pshape_to_rank_in_num_heads_dim(input_layout.get_partial_shape()));
+        if (input_layout.data_padding) {
+            input_layout.data_padding = extend_padding_to_rank_in_num_heads_dim(input_layout.data_padding, input_rank);
+        }
     }
 
     auto& output_layout = updated_impl_params.output_layouts[0];

src/plugins/intel_gpu/src/kernel_selector/cl_kernels/broadcast_gpu_ref.cl

@@ -258,7 +258,11 @@ inline uint FUNC(get_idx_pos)(OPTIONAL_SHAPE_INFO_ARG uint out_b, uint out_f, ui
 KERNEL(broadcast_gpu_ref)(
     OPTIONAL_SHAPE_INFO_ARG
     const __global INPUT0_TYPE* input,
-    __global OUTPUT_TYPE* output)
+    __global OUTPUT_TYPE* output
+#if HAS_FUSED_OPS_DECLS
+    , FUSED_OPS_DECLS
+#endif
+)
 {
 #if SAME_RANK_PLAIN_FORMAT == 1
     const bool use_opt_code = INPUT0_SIZE_X == OUTPUT_SIZE_X && INPUT0_SIZE_Y != OUTPUT_SIZE_Y
@@ -319,11 +323,20 @@ KERNEL(broadcast_gpu_ref)(
         if (remained_y < y_stride)
             y_nums += remained_y;
 
+        #if HAS_FUSED_OPS
+            OUTPUT_TYPE res = 0;
+        #endif
         if (OUTPUT_SIZE_X < VEC_SIZE) {
             uint output_idx = out_pos;
-            unroll_for(uint j = 0; j < y_nums; j++) {
-                unroll_for(uint i = 0; i < x_stride; i++) {
-                    output[output_idx + i] = TO_OUTPUT_TYPE(input[idx_pos + i]);
+            unroll_for(uint i = 0; i < y_nums; i++) {
+                unroll_for(uint offset = 0; offset < x_stride; offset++) {
+                    #if HAS_FUSED_OPS
+                        res = TO_OUTPUT_TYPE(input[idx_pos + offset]);
+                        FUSED_OPS
+                        output[output_idx + offset] = FUSED_OPS_RESULT;
+                    #else
+                        output[output_idx + offset] = TO_OUTPUT_TYPE(input[idx_pos + offset]);
+                    #endif
                 }
                 output_idx += OUTPUT_SIZE_X;
             }
@@ -332,19 +345,35 @@ KERNEL(broadcast_gpu_ref)(
             INPUT0_VTYPE input_vec = VLOAD(0, &input[idx_pos]);
             unroll_for(uint i = 0; i < y_nums; i++) {
                 OUTPUT_VTYPE out_v;
-                for (int j = 0; j < VEC_SIZE; ++j)
-                    out_v[j] = TO_OUTPUT_TYPE(input_vec[j]);
+                for (int offset = 0; offset < VEC_SIZE; ++offset) {
+                    #if HAS_FUSED_OPS
+                        res = TO_OUTPUT_TYPE(input_vec[offset]);
+                        FUSED_OPS
+                        out_v[offset] = FUSED_OPS_RESULT;
+                    #else
+                        out_v[offset] = TO_OUTPUT_TYPE(input_vec[offset]);
+                    #endif
+                }
                 VSTORE(out_v, 0, &output[output_idx]);
                 output_idx += OUTPUT_SIZE_X;
             }
 
             if (gdim0 < x_leftovers) {
                 INPUT0_TYPE input_val = input[idx_pos + x_stride];
+                #if HAS_FUSED_OPS
+                    res = TO_OUTPUT_TYPE(input_val);
+                #endif
 
-                output_idx = out_pos;
+                uint offset = x_stride;
                 unroll_for(uint i = 0; i < y_nums; i++) {
-                    output[output_idx + x_stride] = TO_OUTPUT_TYPE(input_val);
-                    output_idx += OUTPUT_SIZE_X;
+                    #if HAS_FUSED_OPS
+                        FUSED_OPS
+                        output[out_pos + offset] = FUSED_OPS_RESULT;
+                        offset += OUTPUT_SIZE_X;
+                    #else
+                        output[out_pos + offset] = TO_OUTPUT_TYPE(input_val);
+                        offset += OUTPUT_SIZE_X;
+                    #endif
                 }
             }
         }
@@ -379,7 +408,14 @@ KERNEL(broadcast_gpu_ref)(
         const uint out_pos = OUTPUT_GET_INDEX(out_b, out_f, out_y, out_x);
         const uint idx_pos = FUNC_CALL(get_idx_pos)(OPTIONAL_SHAPE_INFO_TENSOR out_b, out_f, out_y, out_x);
 #endif
-        output[out_pos] = TO_OUTPUT_TYPE(input[idx_pos]);
+        #if HAS_FUSED_OPS
+            uint offset = 0;
+            OUTPUT_TYPE res = TO_OUTPUT_TYPE(input[idx_pos]);
+            FUSED_OPS
+            output[out_pos] = FUSED_OPS_RESULT;
+        #else
+            output[out_pos] = TO_OUTPUT_TYPE(input[idx_pos]);
+        #endif
     }
 }
 

src/plugins/intel_gpu/src/kernel_selector/kernels/broadcast/broadcast_kernel_base.cpp

@@ -32,6 +32,24 @@ JitConstants BroadcastKernelBase::GetJitConstants(const broadcast_params& params
     jit.AddConstants({MakeJitConstant("VEC_SIZE", VEC_SIZE)});
     jit.AddConstants({MakeJitConstant("Y_BLOCKS", Y_BLOCKS)});
     jit.AddConstants({MakeJitConstant("SAME_RANK_PLAIN_FORMAT", is_same_planar_format(in_layout, out_layout))});
+
+    // Fused post_ops
+    if (!params.fused_ops.empty()) {
+        kernel_selector::Datatype input_dt = params.outputs[0].GetDType();
+        std::vector<std::string> idx_order;
+        if (DataTensor::ChannelsCount(params.outputs[0].GetLayout()) == 4) {
+            idx_order = {"out_b", "out_f", "out_y", "out_x + offset"};
+        } else if (DataTensor::ChannelsCount(params.outputs[0].GetLayout()) == 5) {
+            idx_order = {"out_b", "out_f", "out_z", "out_y", "out_x + offset"};
+        } else if (DataTensor::ChannelsCount(params.outputs[0].GetLayout()) == 6) {
+            idx_order = {"out_b", "out_f", "out_w", "out_z", "out_y", "out_x + offset"};
+        }
+
+        FusedOpsConfiguration conf = {"", idx_order, "res", input_dt, 1};
+
+        jit.Merge(MakeFusedOpsJitConstants(params, {conf}));
+    }
+
     return jit;
 }
 
@@ -148,7 +166,7 @@ KernelsData BroadcastKernelBase::GetCommonKernelsData(const Params& params) cons
                      false,
                      false,
                      1,
-                     0,
+                     GetFusedPrimitiveInputsCount(params),
                      1,
                      prim_params.is_shape_agnostic);
 

src/plugins/intel_gpu/src/kernel_selector/kernels/broadcast/broadcast_kernel_ref.h

@@ -15,7 +15,9 @@ class BroadcastKernelRef : public BroadcastKernelBase {
     KernelsPriority GetKernelsPriority(const Params& params) const override;
     ParamsKey GetSupportedKey() const override;
     std::vector<FusedOpType> GetSupportedFusedOps() const override {
-        return { FusedOpType::REORDER };
+        return { FusedOpType::ELTWISE,
+                 FusedOpType::REORDER
+        };
     }
 };
 }  // namespace kernel_selector

src/plugins/intel_gpu/tests/unit/fusions/broadcast_fusion_test.cpp

@@ -0,0 +1,95 @@
+// Copyright (C) 2018-2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "test_utils.h"
+#include "fusion_test_common.hpp"
+
+#include <intel_gpu/primitives/input_layout.hpp>
+#include <intel_gpu/primitives/broadcast.hpp>
+#include <intel_gpu/primitives/eltwise.hpp>
+
+#include <cmath>
+
+using namespace cldnn;
+using namespace ::tests;
+
+namespace {
+struct broadcast_test_params {
+    ov::PartialShape input_size1;  // input for broadcast
+    ov::PartialShape input_size2;  // other input connected to output of broadcast
+    data_types input_type1;        // input data-type of 'input_size1'
+    data_types input_type2;
+    format input_format;
+    data_types default_type;
+    format default_format;
+    ov::AxisSet broadcast_axes;
+    size_t expected_fused_primitives;
+    size_t expected_not_fused_primitives;
+};
+
+class BroadcastFusingTest : public ::BaseFusingTest<broadcast_test_params> {
+public:
+    void execute(broadcast_test_params& p, bool count_reorder = false) {
+        auto input_prim = get_mem(get_input_layout1(p));
+        auto input_prim2 = get_mem(get_input_layout2(p));
+
+        network network_not_fused(this->engine, this->topology_non_fused, cfg_not_fused);
+        network network_fused(this->engine, this->topology_fused, cfg_fused);
+
+        auto inputs = network_fused.get_input_ids();
+        if (std::find(inputs.begin(), inputs.end(), "input") != inputs.end()) {
+            network_fused.set_input_data("input", input_prim);
+            network_not_fused.set_input_data("input", input_prim);
+        }
+        if (std::find(inputs.begin(), inputs.end(), "input2") != inputs.end()) {
+            network_fused.set_input_data("input2", input_prim2);
+            network_not_fused.set_input_data("input2", input_prim2);
+        }
+
+        compare(network_not_fused, network_fused, p, count_reorder);
+    }
+
+    layout get_input_layout1(broadcast_test_params& p) {
+        return layout{ p.input_size1, p.input_type1, p.input_format };
+    }
+
+    layout get_input_layout2(broadcast_test_params& p) {
+        return layout{ p.input_size2, p.input_type2, p.input_format };
+    }
+};
+}  // namespace
+
+#define CASE_BROADCAST_FP16_1         { 1, 16, 4, 4 }, { 2, 16, 4, 4 },        data_types::f16, data_types::f16, format::bfyx,           data_types::f16,  format::bfyx,          {0}
+#define CASE_BROADCAST_FP16_2         { 2, 1,  4, 4, 4 }, { 2, 16, 4, 4, 4 },  data_types::f16, data_types::f16, format::bfzyx,          data_types::f16,  format::bfzyx,         {1}
+#define CASE_BROADCAST_FP16_3         { 2, 16, 4, 4, 1 }, { 2, 16, 4, 4, 8 },  data_types::f16, data_types::f16, format::bfzyx,          data_types::f16,  format::bfzyx,         {4}
+
+#define CASE_BROADCAST_FP16_1_BLK     { 2, 16, 4, 1 }, { 2, 16, 4, 4 },        data_types::f16, data_types::f16, format::b_fs_yx_fsv16,  data_types::f16,  format::bfyx,          {3}
+#define CASE_BROADCAST_FP16_2_BLK     { 1, 16, 4, 4 }, { 2, 16, 4, 4 },        data_types::f16, data_types::f16, format::b_fs_yx_fsv16,  data_types::f16,  format::bfyx,          {0}
+#define CASE_BROADCAST_FP16_3_BLK     { 2, 16, 4, 1 }, { 2, 16, 4, 4 },        data_types::u8, data_types::i8,   format::b_fs_yx_fsv32,  data_types::f16,  format::bfyx,          {3}
+
+class broadcast_fused_prims : public BroadcastFusingTest {};
+TEST_P(broadcast_fused_prims, broadcast_activation_with_broadcast) {
+    auto p = GetParam();
+    create_topologies(
+        input_layout("input", get_input_layout1(p)),
+        input_layout("input2", get_input_layout2(p)),
+        broadcast("broadcast", input_info("input"), get_input_layout2(p).get_shape(), ov::AxisSet(p.broadcast_axes), ov::op::BroadcastType::NUMPY),
+        eltwise("eltwise", { input_info("broadcast"), input_info("input2") }, eltwise_mode::sum, p.default_type),
+        activation("activation", input_info("eltwise"), activation_func::abs),
+        reorder("out", input_info("activation"), p.default_format, data_types::f32)
+    );
+
+    tolerance = default_tolerance(p.input_type1);
+    execute(p);
+}
+
+INSTANTIATE_TEST_SUITE_P(fusings_gpu, broadcast_fused_prims, ::testing::ValuesIn(std::vector<broadcast_test_params>{
+    broadcast_test_params{ CASE_BROADCAST_FP16_1, 4, 5 },
+    broadcast_test_params{ CASE_BROADCAST_FP16_2, 4, 5 },
+    broadcast_test_params{ CASE_BROADCAST_FP16_3, 4, 5 },
+
+    broadcast_test_params{ CASE_BROADCAST_FP16_1_BLK, 4, 5 },
+    broadcast_test_params{ CASE_BROADCAST_FP16_2_BLK, 4, 5 },
+    broadcast_test_params{ CASE_BROADCAST_FP16_3_BLK, 4, 5 },
+}));

src/plugins/intel_gpu/thirdparty/onednn_gpu

@@ -879,7 +879,12 @@ void ov::npuw::LLMInferRequest::infer() {
     if (input_ids->get_shape()[INPUT_IDS_SEQ_LEN_DIM] > 1 && position_ids->data<int64_t>()[0] == m_first_position_id) {
         infer_prefill(input_ids, attention_mask, position_ids, token_type_ids);
     } else {
-        trim_kvcache_for_speculative_decoding(position_ids);
+        auto& kvcache_desc = m_npuw_llm_compiled_model->m_kvcache_desc;
+        // Need to reconsider the solution. Some model like Qwen2.5VL, doesn't use speculative decoding,
+        // but it may have repeated position ids, then it will trigger kvcache trim and cause AC issue.
+        if (kvcache_desc.max_generation_token_len > 1) {
+            trim_kvcache_for_speculative_decoding(position_ids);
+        }
         infer_generate(input_ids, attention_mask, position_ids, token_type_ids);
     }
 }