[CPU] migrate onednn3.6 (#27957)

1. migrate to oneDNN3.6
  - apply x64 patch
  - apply arm patch
  - apply rv64 patch
 
2. fix conflicts in OpenVINO
  - adapt `jit_uni_eltwise_injector` constructor
  - adapt `dnnl::impl::md2fmt_str` func

3. track by tickets
  - MFDNN-12818
  - MFDNN-12919
  - MFDNN-12924
 
4. oneDNN link
- branch:
https://github.com/openvinotoolkit/oneDNN/tree/xiuchuan/dev-v3.6

Author: azhai219

src/plugins/intel_cpu/src/emitters/plugin/x64/jit_dnnl_emitters.cpp

@@ -46,13 +46,18 @@ jit_dnnl_emitter::jit_dnnl_emitter(jit_generator* host,
 void jit_dnnl_emitter::set_injector() {
     if (host_isa_ == cpu::x64::sse41) {
         eltwise_injector_sse42 =
-            std::make_shared<jit_uni_eltwise_injector_f32<cpu::x64::sse41>>(h, kind, alpha, beta, 1.f);
+            std::make_shared<jit_uni_eltwise_injector<cpu::x64::sse41>>(h, kind, alpha, beta, 1.f, data_type::f32);
     } else if (host_isa_ == cpu::x64::avx2) {
         eltwise_injector_avx2 =
-            std::make_shared<jit_uni_eltwise_injector_f32<cpu::x64::avx2>>(h, kind, alpha, beta, 1.f);
+            std::make_shared<jit_uni_eltwise_injector<cpu::x64::avx2>>(h, kind, alpha, beta, 1.f, data_type::f32);
     } else if (host_isa_ == cpu::x64::avx512_core) {
         eltwise_injector_avx512_core =
-            std::make_shared<jit_uni_eltwise_injector_f32<cpu::x64::avx512_core>>(h, kind, alpha, beta, 1.f);
+            std::make_shared<jit_uni_eltwise_injector<cpu::x64::avx512_core>>(h,
+                                                                              kind,
+                                                                              alpha,
+                                                                              beta,
+                                                                              1.f,
+                                                                              data_type::f32);
     } else {
         OV_CPU_JIT_EMITTER_THROW("Unsupported ISA ", host_isa_);
     }

src/plugins/intel_cpu/src/emitters/plugin/x64/jit_dnnl_emitters.hpp

@@ -42,11 +42,9 @@ class jit_dnnl_emitter : public jit_emitter {
     float alpha{0.f};
     float beta{0.f};
 
-    std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector_f32<dnnl::impl::cpu::x64::sse41>>
-        eltwise_injector_sse42;
-    std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector_f32<dnnl::impl::cpu::x64::avx2>>
-        eltwise_injector_avx2;
-    std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector_f32<dnnl::impl::cpu::x64::avx512_core>>
+    std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector<dnnl::impl::cpu::x64::sse41>> eltwise_injector_sse42;
+    std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector<dnnl::impl::cpu::x64::avx2>> eltwise_injector_avx2;
+    std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector<dnnl::impl::cpu::x64::avx512_core>>
         eltwise_injector_avx512_core;
 
 private:

src/plugins/intel_cpu/src/nodes/bin_conv.cpp

@@ -66,11 +66,12 @@ struct jit_uni_bin_conv_kernel_f32 : public jit_uni_bin_conv_kernel, public jit_
         for (int i = 0; i < end_idx; i++) {
             auto& post_op = p.entry_[i];
             if (post_op.is_eltwise()) {
-                eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector_f32<isa>>(this,
-                                                                                                post_op.eltwise,
-                                                                                                true,
-                                                                                                eltwise_reserved,
-                                                                                                mask_post_op_reserved));
+                eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector<isa>>(this,
+                                                                                            post_op.eltwise,
+                                                                                            data_type::f32,
+                                                                                            true,
+                                                                                            eltwise_reserved,
+                                                                                            mask_post_op_reserved));
             } else if (post_op.is_depthwise()) {
                 depthwise_injectors.push_back(
                     std::make_shared<jit_uni_depthwise_injector_f32<isa>>(this, post_op, mask_post_op_reserved));
@@ -217,7 +218,7 @@ struct jit_uni_bin_conv_kernel_f32 : public jit_uni_bin_conv_kernel, public jit_
 
     Xbyak::Label l_table;
 
-    nstl::vector<std::shared_ptr<jit_uni_eltwise_injector_f32<isa>>> eltwise_injectors;
+    nstl::vector<std::shared_ptr<jit_uni_eltwise_injector<isa>>> eltwise_injectors;
     nstl::vector<std::shared_ptr<jit_uni_depthwise_injector_f32<isa>>> depthwise_injectors;
 
     void cvt2ps(dnnl::memory::data_type type_in, Vmm vmm_in, const Xbyak::Operand& op, bool scalar_load) {

src/plugins/intel_cpu/src/nodes/common/softmax.cpp

@@ -16,6 +16,7 @@
 #include "utils/bfloat16.hpp"
 
 using namespace dnnl;
+using namespace dnnl::impl;
 using namespace dnnl::impl::cpu;
 using namespace dnnl::impl::cpu::x64;
 using namespace dnnl::impl::utils;
@@ -68,7 +69,7 @@ struct jit_uni_softmax_kernel_f32 : public jit_uni_softmax_kernel, public jit_ge
 
     void generate() override {
         exp_injector.reset(
-            new jit_uni_eltwise_injector_f32<isa>(this, dnnl::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.0f));
+            new jit_uni_eltwise_injector<isa>(this, dnnl::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.0f, data_type::f32));
 
         if (mayiuse(avx512_core))
             uni_vcvtneps2bf16.reset(new jit_uni_vcvtneps2bf16(this, isa));
@@ -200,7 +201,7 @@ struct jit_uni_softmax_kernel_f32 : public jit_uni_softmax_kernel, public jit_ge
 
     std::unique_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16;
 
-    std::shared_ptr<jit_uni_eltwise_injector_f32<isa>> exp_injector;
+    std::shared_ptr<jit_uni_eltwise_injector<isa>> exp_injector;
 
     jit_softmax_config_params jcp_;
 

src/plugins/intel_cpu/src/nodes/interpolate.cpp

@@ -73,11 +73,12 @@ struct jit_uni_interpolate_kernel_f32 : public jit_uni_interpolate_kernel, publi
         for (int i = 0; i < p.len(); i++) {
             auto& post_op = p.entry_[i];
             if (post_op.is_eltwise()) {
-                eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector_f32<isa>>(this,
-                                                                                                post_op.eltwise.alg,
-                                                                                                post_op.eltwise.alpha,
-                                                                                                post_op.eltwise.beta,
-                                                                                                1.f));
+                eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector<isa>>(this,
+                                                                                            post_op.eltwise.alg,
+                                                                                            post_op.eltwise.alpha,
+                                                                                            post_op.eltwise.beta,
+                                                                                            1.f,
+                                                                                            data_type::f32));
             } else if (post_op.is_depthwise()) {
                 depthwise_injectors.push_back(std::make_shared<jit_uni_depthwise_injector_f32<isa>>(this, post_op));
             } else if (post_op.is_quantization()) {
@@ -275,7 +276,7 @@ struct jit_uni_interpolate_kernel_f32 : public jit_uni_interpolate_kernel, publi
     std::vector<size_t> store_pool_vec_idxs;
     std::vector<size_t> load_pool_gpr_idxs;
 
-    std::vector<std::shared_ptr<jit_uni_eltwise_injector_f32<isa>>> eltwise_injectors;
+    std::vector<std::shared_ptr<jit_uni_eltwise_injector<isa>>> eltwise_injectors;
     std::vector<std::shared_ptr<jit_uni_depthwise_injector_f32<isa>>> depthwise_injectors;
     std::vector<std::shared_ptr<jit_uni_quantization_injector_f32<isa>>> quantization_injectors;
 

src/plugins/intel_cpu/src/nodes/kernels/x64/brgemm_kernel.cpp

@@ -289,7 +289,7 @@ void BrgemmKernel::init_brgemm_copy_b(
     brgemm_matmul_conf_t brgCopyKernelConf;
     brgCopyKernelConf.src_dt = is_avx_f16_only ? dnnl_data_type_t::dnnl_f32 : dt_in0;
     brgCopyKernelConf.wei_dt = is_avx_f16_only ? dnnl_data_type_t::dnnl_f32 : dt_in1;
-    brgCopyKernelConf.orig_wei_dt = dt_in1;
+    brgCopyKernelConf.orig_wei_dt = static_cast<dnnl_data_type_t>(DnnlExtensionUtils::ElementTypeToDataType(inType));
     brgCopyKernelConf.wei_n_blk = N_blk;
     brgCopyKernelConf.wei_tag = transpose ? dnnl_ba : dnnl_ab;
     brgCopyKernelConf.copy_B_wei_stride = copy_B_wei_stride;

src/plugins/intel_cpu/src/nodes/kernels/x64/mlp_kernel.cpp

@@ -550,12 +550,13 @@ void GateUpCombine::generate() {
     const auto zmm_up = zmm0;
     const auto ymm_dst = ymm5;
 
-    auto injector = std::make_shared<jit_uni_eltwise_injector_f32<dnnl::impl::cpu::x64::avx512_core>>(
+    auto injector = std::make_shared<jit_uni_eltwise_injector<dnnl::impl::cpu::x64::avx512_core>>(
         this,
         m_act_alg,
         1.f,
         1.0f,
         1.f,
+        data_type::f32,
         true,                               // save_state, true due to additional r15 is used.
         Xbyak::Reg64(Xbyak::Operand::R10),  // p_table
         Xbyak::Opmask(1),                   // k_mask

src/plugins/intel_cpu/src/nodes/kernels/x64/non_max_suppression.cpp

@@ -6,6 +6,7 @@
 
 #include "utils/general_utils.h"
 
+using namespace dnnl::impl;
 using namespace dnnl::impl::cpu;
 
 #define GET_OFF(field) offsetof(NmsCallArgs, field)
@@ -20,7 +21,7 @@ void NonMaxSuppression<isa>::generate() {
     load_scalar_emitter.reset(new jit_load_emitter(this, isa, ov::element::f32, ov::element::f32, scalar_step));
 
     exp_injector.reset(
-        new x64::jit_uni_eltwise_injector_f32<isa>(this, dnnl::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.f));
+        new x64::jit_uni_eltwise_injector<isa>(this, dnnl::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.f, data_type::f32));
 
     this->preamble();
 

src/plugins/intel_cpu/src/nodes/kernels/x64/non_max_suppression.hpp

@@ -110,7 +110,7 @@ class NonMaxSuppression : public JitKernel<NmsCompileParams, NmsCallArgs> {
     Xbyak::Opmask k_mask = Xbyak::Opmask(7);
     Xbyak::Opmask k_mask_one = Xbyak::Opmask(6);
 
-    std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector_f32<isa>> exp_injector;
+    std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector<isa>> exp_injector;
 
     inline void hard_nms();
 

src/plugins/intel_cpu/src/nodes/mvn.cpp

@@ -949,11 +949,12 @@ struct jit_uni_mvn_kernel_f32 : public jit_uni_mvn_kernel, public jit_generator
         for (int i = 0; i < p.len(); i++) {
             auto& post_op = p.entry_[i];
             if (post_op.is_eltwise()) {
-                eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector_f32<isa>>(this,
-                                                                                                post_op.eltwise.alg,
-                                                                                                post_op.eltwise.alpha,
-                                                                                                post_op.eltwise.beta,
-                                                                                                post_op.eltwise.scale));
+                eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector<isa>>(this,
+                                                                                            post_op.eltwise.alg,
+                                                                                            post_op.eltwise.alpha,
+                                                                                            post_op.eltwise.beta,
+                                                                                            post_op.eltwise.scale,
+                                                                                            data_type::f32));
             } else if (post_op.is_depthwise()) {
                 depthwise_injectors.push_back(std::make_shared<jit_uni_depthwise_injector_f32<isa>>(this, post_op));
             } else if (post_op.is_quantization()) {
@@ -1093,7 +1094,7 @@ struct jit_uni_mvn_kernel_f32 : public jit_uni_mvn_kernel, public jit_generator
 
     const int tile_size[kTileNum] = {8, 4, 2, 1};
 
-    std::vector<std::shared_ptr<jit_uni_eltwise_injector_f32<isa>>> eltwise_injectors;
+    std::vector<std::shared_ptr<jit_uni_eltwise_injector<isa>>> eltwise_injectors;
     std::vector<std::shared_ptr<jit_uni_depthwise_injector_f32<isa>>> depthwise_injectors;
     std::vector<std::shared_ptr<jit_uni_quantization_injector_f32<isa>>> quantization_injectors;