[camb]tyf/change permute (DeepLink-org#876)

* change permute

Author: Bonbon-Tang

adaptor/codegen/gen.py

@@ -38,7 +38,7 @@
     "NLC": "diopiMemoryFormat_t::ChannelsLast1d",
     "NHWC": "diopiMemoryFormat_t::ChannelsLast",
     "NDHWC": "diopiMemoryFormat_t::ChannelsLast3d",
-    "UD": "diopiMemoryFormat_t::Preserve",
+    "ND": "diopiMemoryFormat_t::Preserve",
 }
 
 
@@ -443,7 +443,7 @@ def analysis_configs(config: List[dict], funcs_info: dict) -> dict:
                         or layout == "NCL"
                         or layout == "NDHWC"
                         or layout == "NCDHW"
-                        or layout == "UD"
+                        or layout == "ND"
                     ):
                         op_layouts.append(layout)
                     else:

adaptor/csrc/convert.cpp

@@ -1,7 +1,12 @@
 #include "convert.hpp"
 
-bool denseCheckAdaptor(diopiSize_t shape, diopiSize_t stride) {
+bool denseCheck(diopiSize_t shape, diopiSize_t stride) {
     int dim = shape.len;
+
+    if (isContiguous(shape, stride, diopiMemoryFormat_t::Contiguous)) {
+        return true;
+    }
+
     std::vector<std::pair<int64_t, int64_t>> stridesSizes(dim, std::pair<int64_t, int64_t>(1, 1));
 
     for (int i = 0; i < dim; i++) {

adaptor/csrc/convert.hpp

@@ -13,7 +13,7 @@
 #include <ostream>
 #include <vector>
 
-bool denseCheckAdaptor(diopiSize_t shape, diopiSize_t stride);
+bool denseCheck(diopiSize_t shape, diopiSize_t stride);
 
 std::vector<int64_t> calcStrides(diopiSize_t size, diopiMemoryFormat_t format = diopiMemoryFormat_t::Contiguous);
 
@@ -111,7 +111,7 @@ ConvertType castImpl(diopiContextHandle_t ctx, T src, T* dst, std::vector<diopiM
     diopiSize_t dstStride = srcStride;
     diopiSize_t dstSize = srcSize;
     if (!targetMemoryFormats.empty()) {
-        if (!denseCheckAdaptor(srcSize, srcStride) && supportMemoryFormats[0] == diopiMemoryFormat_t::Preserve) {
+        if (!denseCheck(srcSize, srcStride) && supportMemoryFormats[0] == diopiMemoryFormat_t::Preserve) {
             targetMemoryFormats.push_back(diopiMemoryFormat_t::Preserve);
             needConvertMemoryFormat = true;
         }

impl/camb/common/clone.cpp

@@ -10,6 +10,11 @@ namespace impl {
 namespace camb {
 diopiError_t clone(diopiContextHandle_t ctx, const DiopiTensor& inTensor, DiopiTensor& outTensor, diopiMemoryFormat_t memoryFormat) {
     cnnlHandle_t handle = cnnlHandlePool.get(ctx);
+    if (!denseCheck(const_cast<DiopiTensor&>(inTensor))) {
+        DiopiTensor denseOut;
+        toDense(ctx, const_cast<DiopiTensor&>(inTensor), denseOut);
+        const_cast<DiopiTensor&>(inTensor) = denseOut;
+    }
     if (memoryFormat == diopiMemoryFormat_t::Preserve) {
         // torch.preserve_format: Used in functions like clone to preserve the memory format of the input tensor.
         // If input tensor is allocated in dense non-overlapping memory, the output tensor strides will be copied from the input.
@@ -19,9 +24,10 @@ diopiError_t clone(diopiContextHandle_t ctx, const DiopiTensor& inTensor, DiopiT
     } else {
         outTensor = requiresTensor(ctx, inTensor.shape(), inTensor.dtype(), memoryFormat);
     }
+
     if (inTensor.shape() == outTensor.shape() && inTensor.dim() != 0 && inTensor.dtype() != diopi_dtype_float64 && inTensor.dtype() == outTensor.dtype() &&
-        denseCheck(inTensor) && denseCheck(outTensor) && (outTensor.isContiguous() || inTensor.isContiguous())) {
-        DIOPI_CALL(contiguousOut(ctx, const_cast<DiopiTensor&>(inTensor), outTensor));
+        denseCheck(outTensor)) {
+        DIOPI_CALL(permuteCopy(ctx, const_cast<DiopiTensor&>(inTensor), outTensor));
         return diopiSuccess;
     }
     CnnlTensorDesc inTensorDesc(inTensor, CNNL_LAYOUT_ARRAY);

impl/camb/common/common.hpp

@@ -34,7 +34,7 @@ bool broadcast(DiopiTensor inputTensor, const std::vector<int64_t>& targetShape,
 
 diopiError_t contiguous(diopiContextHandle_t ctx, DiopiTensor& src, diopiMemoryFormat_t memoryFormat = diopiMemoryFormat_t::Contiguous);
 
-diopiError_t contiguousOut(diopiContextHandle_t ctx, DiopiTensor& src, DiopiTensor& dest);
+diopiError_t permuteCopy(diopiContextHandle_t ctx, DiopiTensor& src, DiopiTensor& dest);
 
 diopiError_t contiguous(diopiContextHandle_t ctx, DiopiTensor& src, diopiMemoryFormat_t memoryFormat, cnnlTensorLayout_t layoutIn,
                         cnnlTensorLayout_t layoutOut);
@@ -59,6 +59,10 @@ bool isSlice(const DiopiTensor& src);
 
 bool isSparse(const DiopiTensor& src);
 
+diopiError_t permuteTensor(DiopiTensor& t, const std::vector<int32_t>& order);
+
+diopiError_t getPermuteOrder(const DiopiTensor& src, std::vector<int32_t>& orderOut, std::vector<int32_t>& reverseOrder);
+
 diopiError_t getDenseStride(const DiopiTensor& src, std::vector<int64_t>& dstStride);
 
 diopiError_t sliceToDense(diopiContextHandle_t ctx, DiopiTensor& src, DiopiTensor& dst);

impl/camb/common/contiguous.cpp

@@ -24,19 +24,7 @@ static diopiError_t transpose(diopiContextHandle_t& ctx, DiopiTensor& in, DiopiT
     return diopiSuccess;
 }
 
-// static diopiError_t calTensordiopiMemoryFormat_t(const DiopiTensor& tensor, diopiMemoryFormat_t& memoryFormatOut) {
-//     if (tensor.isContiguous(diopiMemoryFormat_t::ChannelsLast)) {
-//         memoryFormatOut = diopiMemoryFormat_t::ChannelsLast;
-//     } else if (tensor.isContiguous(diopiMemoryFormat_t::ChannelsLast3d)) {
-//         memoryFormatOut = diopiMemoryFormat_t::ChannelsLast3d;
-//     } else if (tensor.isContiguous(diopiMemoryFormat_t::Contiguous)) {
-//         memoryFormatOut = diopiMemoryFormat_t::Contiguous;
-//     } else {
-//         return diopiNoImplement;
-//     }
-//     return diopiSuccess;
-// }
-static diopiError_t getPermuteOrder(const DiopiTensor& src, std::vector<int32_t>& orderOut, std::vector<int32_t>& reverseOrder) {
+diopiError_t getPermuteOrder(const DiopiTensor& src, std::vector<int32_t>& orderOut, std::vector<int32_t>& reverseOrder) {
     if (src.isContiguous()) {
         orderOut.resize(src.dim());
         for (int i = 0; i < src.dim(); ++i) {
@@ -59,6 +47,7 @@ static diopiError_t getPermuteOrder(const DiopiTensor& src, std::vector<int32_t>
         stridesSizes[i] = std::pair<int, int>(inputStrides[i], inputSizes[i]);
     }
 
+    // shape:2,3,4,5 stride:60,1,15,3 -> orderOut: 0,3,1,2, reverseOrder: 0,2,3,1
     sort(stridesSizes.begin(), stridesSizes.end(), [](std::pair<int, int> a, std::pair<int, int> b) { return a.first > b.first; });
     for (int i = 0; i < dim; ++i) {
         auto pair = stridesSizes[i];
@@ -83,73 +72,6 @@ static diopiError_t getPermuteOrder(const DiopiTensor& src, std::vector<int32_t>
     return diopiSuccess;
 }
 
-static diopiError_t calOrderAndSrcMemoryFormat(const DiopiTensor& src, diopiMemoryFormat_t destMemoryFormat, diopiMemoryFormat_t& srcMemoryFormatOut,
-                                               std::vector<int32_t>& orderOut, std::vector<int32_t>& reverseOrder) {
-    if (src.isContiguous(destMemoryFormat)) {
-        srcMemoryFormatOut = destMemoryFormat;
-        orderOut.resize(src.dim());
-        for (int i = 0; i < src.dim(); ++i) {
-            orderOut[i] = i;
-        }
-        reverseOrder = orderOut;
-        return diopiSuccess;
-    }
-    if (src.isContiguous(diopiMemoryFormat_t::ChannelsLast1d) && destMemoryFormat == diopiMemoryFormat_t::Contiguous) {
-        if (src.dim() != 3) {
-            setLastErrorString("the dim of the tensor should be 4, but now is %d.", src.dim());
-            return diopiNoImplement;
-        }
-        srcMemoryFormatOut = diopiMemoryFormat_t::ChannelsLast1d;
-        orderOut = {0, 2, 1};
-        reverseOrder = {0, 2, 1};
-    } else if (src.isContiguous(diopiMemoryFormat_t::Contiguous) && destMemoryFormat == diopiMemoryFormat_t::ChannelsLast1d) {
-        if (src.dim() != 3) {
-            setLastErrorString("the dim of the tensor should be 4, but now is %d.", src.dim());
-            return diopiNoImplement;
-        }
-        srcMemoryFormatOut = diopiMemoryFormat_t::Contiguous;
-        orderOut = {0, 2, 1};
-        reverseOrder = {0, 2, 1};
-    } else if (src.isContiguous(diopiMemoryFormat_t::ChannelsLast) && destMemoryFormat == diopiMemoryFormat_t::Contiguous) {
-        if (src.dim() != 4) {
-            setLastErrorString("the dim of the tensor should be 4, but now is %d.", src.dim());
-            return diopiNoImplement;
-        }
-        srcMemoryFormatOut = diopiMemoryFormat_t::ChannelsLast;
-        orderOut = {0, 3, 1, 2};
-        reverseOrder = {0, 2, 3, 1};
-    } else if (src.isContiguous(diopiMemoryFormat_t::Contiguous) && destMemoryFormat == diopiMemoryFormat_t::ChannelsLast) {
-        if (src.dim() != 4) {
-            setLastErrorString("the dim of the tensor should be 4, but now is %d.", src.dim());
-            return diopiNoImplement;
-        }
-        srcMemoryFormatOut = diopiMemoryFormat_t::Contiguous;
-        orderOut = {0, 2, 3, 1};
-        reverseOrder = {0, 3, 1, 2};
-    } else if (src.isContiguous(diopiMemoryFormat_t::Contiguous) && destMemoryFormat == diopiMemoryFormat_t::ChannelsLast3d) {
-        if (src.dim() != 5) {
-            setLastErrorString("the dim of the tensor should be 5, but now is %d.", src.dim());
-            return diopiNoImplement;
-        }
-        srcMemoryFormatOut = diopiMemoryFormat_t::Contiguous;
-        orderOut = {0, 2, 3, 4, 1};
-        reverseOrder = {0, 4, 1, 2, 3};
-    } else if (src.isContiguous(diopiMemoryFormat_t::ChannelsLast3d) && destMemoryFormat == diopiMemoryFormat_t::Contiguous) {
-        if (src.dim() != 5) {
-            setLastErrorString("the dim of the tensor should be 5, but now is %d.", src.dim());
-            return diopiNoImplement;
-        }
-        srcMemoryFormatOut = diopiMemoryFormat_t::ChannelsLast3d;
-        orderOut = {0, 4, 1, 2, 3};
-        reverseOrder = {0, 2, 3, 4, 1};
-    } else {
-        // convert to contiguous format
-        srcMemoryFormatOut = diopiMemoryFormat_t::Preserve;
-        return diopiSuccess;
-    }
-    return diopiSuccess;
-}
-
 diopiError_t calCnnlLayout(diopiMemoryFormat_t memoryFormat, int64_t dim, cnnlTensorLayout_t& cnnlLayout) {
     switch (memoryFormat) {
         case diopiMemoryFormat_t::ChannelsLast1d:
@@ -234,68 +156,61 @@ diopiError_t contiguous(diopiContextHandle_t ctx, DiopiTensor& src, diopiMemoryF
 
     int64_t dim = src.dim();
     DIOPI_CHECK(dim <= 8, "only support less than 8d tensor currently");
-    diopiMemoryFormat_t srcMemoryFormat;
-    std::vector<int32_t> order;
-    std::vector<int32_t> reverseOrder;
     DiopiTensor dest;
-    DIOPI_CALL(calOrderAndSrcMemoryFormat(src, memoryFormat, srcMemoryFormat, order, reverseOrder));
-    if (srcMemoryFormat == diopiMemoryFormat_t::Preserve) {
-        DIOPI_CALL(clone(ctx, src, dest, memoryFormat));
-        src = dest;
-        return diopiSuccess;
-    }
-    dest = requiresTensor(ctx, src.shape(), src.dtype(), memoryFormat);
-    // set CNNL_LAYOUT_ARRAY because NLC->NCL failed ( no layout NCL);
-    cnnlTensorLayout_t srcLayout = CNNL_LAYOUT_ARRAY;
-    cnnlTensorLayout_t destLayout = CNNL_LAYOUT_ARRAY;
-
-    std::vector<int64_t> olderDestStride = dest.stride();
-    std::vector<int64_t> olderDestShape = dest.shape();
-    if (memoryFormat != diopiMemoryFormat_t::Contiguous) {
-        DIOPI_CALL(permuteTensor(dest, order));
-    } else {
-        DIOPI_CALL(permuteTensor(src, reverseOrder));
-    }
-    DIOPI_CALL(transpose(ctx, src, dest, srcLayout, destLayout, order));
-    // recovery the shape
-    dest.asStrided(olderDestShape, olderDestStride);
+    DIOPI_CALL(clone(ctx, src, dest, memoryFormat));
     src = dest;
     return diopiSuccess;
 }
 
-// inplace contiguous
-diopiError_t contiguousOut(diopiContextHandle_t ctx, DiopiTensor& src, DiopiTensor& dest) {
+diopiError_t permuteCopy(diopiContextHandle_t ctx, DiopiTensor& src, DiopiTensor& dest) {
+    // using input permute + output permute + cnnltranspose to copy
     DIOPI_CHECK(src.shape() == dest.shape(), "src's shape should be the same as dest's");
     int64_t dim = src.dim();
     DIOPI_CHECK(dim <= 8, "only support less than 8d tensor currently");
-    std::vector<int32_t> order(dim, 0);
-    std::vector<int32_t> reverseOrder(dim, 0);
+    bool srcIsContiguous = src.isContiguous();
+    bool destIsContiguous = dest.isContiguous();
+    std::vector<int32_t> inputOrder(dim, 0);
+    std::vector<int32_t> inputBackOrder(dim, 0);  // permuteTensor(input,inputBackOrder)->contiguous
+    std::vector<int32_t> outputOrder(dim, 0);
+    std::vector<int32_t> outputBackOrder(dim, 0);     // permuteTensor(output,outputBackOrder)->contiguous
+    std::vector<int32_t> inputToOutputOrder(dim, 0);  // into cnnltranspose
+
+    // input shape:2,3,4,5 stride:60,1,15,3 -> inputBackOrder: 0,2,3,1, inputOrder: 0,3,1,2
+    // output shape:2,3,4,5 stride:60,20,1,4 -> outputBackOrder: 0,1,3,2, outputOrder: 0,1,3,2
+    // inputToOutputOrder: 0,2,1,3
+
+    getPermuteOrder(src, inputOrder, inputBackOrder);
+    getPermuteOrder(dest, outputOrder, outputBackOrder);
 
-    if (src.isContiguous()) {
-        getPermuteOrder(dest, reverseOrder, order);
-    } else {
-        getPermuteOrder(src, order, reverseOrder);
-    }
-    // set CNNL_LAYOUT_ARRAY because NLC->NCL failed ( no layout NCL);
     cnnlTensorLayout_t srcLayout = CNNL_LAYOUT_ARRAY;
     cnnlTensorLayout_t destLayout = CNNL_LAYOUT_ARRAY;
 
     std::vector<int64_t> olderDestStride = dest.stride();
     std::vector<int64_t> olderDestShape = dest.shape();
     std::vector<int64_t> olderSrcStride = src.stride();
     std::vector<int64_t> olderSrcShape = src.shape();
-    // if (destMemoryFormat != diopiMemoryFormat_t::Contiguous) {
-    if (src.isContiguous()) {
-        DIOPI_CALL(permuteTensor(dest, order));
-    } else {
-        DIOPI_CALL(permuteTensor(src, reverseOrder));
+
+    // permute to get contiguous tensor
+    if (!destIsContiguous) {
+        DIOPI_CALL(permuteTensor(dest, outputBackOrder));
+    }
+
+    if (!srcIsContiguous) {
+        DIOPI_CALL(permuteTensor(src, inputBackOrder));
+    }
+
+    for (int i = 0; i < dim; ++i) {
+        inputToOutputOrder[i] = inputOrder[outputBackOrder[i]];
     }
-    DIOPI_CALL(transpose(ctx, src, dest, srcLayout, destLayout, order));
+
+    DIOPI_CALL(transpose(ctx, src, dest, srcLayout, destLayout, inputToOutputOrder));
+
     // recovery the shape and strides
-    // if (destMemoryFormat != diopiMemoryFormat_t::Contiguous) {
-    if (src.isContiguous()) {
+    if (!destIsContiguous) {
         dest.asStrided(olderDestShape, olderDestStride);
-    } else {
+    }
+
+    if (!srcIsContiguous) {
         src.asStrided(olderSrcShape, olderSrcStride);
     }
     return diopiSuccess;

impl/camb/common/denseCheck.cpp

@@ -124,6 +124,8 @@ diopiError_t sliceToDense(diopiContextHandle_t ctx, DiopiTensor& src, DiopiTenso
 
     // 得到按照stride从大到小顺序排列的shape，实测这种copy最快，也可以考虑用slice;
     // e.g. shape 128 12 64 197 stride 453888 64 1 2304-> shape 128 197 64 12 stride 453888 2304 64 1
+    // e.g. shape: [128, 768, 14, 14], stride: [151296, 1, 10752, 768]
+    // -> shape: 128 14 14 768, stride:151296, 10752, 768, 1
     std::vector<int64_t> generateShape;
     std::vector<int64_t> generateStride;
     std::vector<int64_t> generateOutStride(dim, 0);

impl/camb/convert_config.yaml

@@ -15,10 +15,10 @@
     layout: NCL, NLC, NCHW, NHWC, NDHWC
 
 - diopiDivScalar:
-    layout: UD
+    layout: ND
 
 - diopiDivScalar:
-    layout: UD
+    layout: ND
 
 - diopiBmm:
     layout: NCL, NLC, NCHW, NHWC, NCDHW, NDHWC
@@ -81,4 +81,4 @@
 #* Ops above are not neccesary to convert format *#
 
 - diopiReluInp:
-    layout: UD
+    layout: ND

impl/camb/functions/copy.cpp

@@ -37,6 +37,7 @@ diopiError_t diopiCopyInp(diopiContextHandle_t ctx, diopiConstTensorHandle_t src
         return diopiSuccess;
     }
     DiopiTensor srcTr(src);
+    DiopiTensor srcTmpTr(src);
     DiopiTensor destTr(dest);
     cnnlHandle_t handle = cnnlHandlePool.get(ctx);
     if (!srcTr.defined()) {
@@ -47,21 +48,27 @@ diopiError_t diopiCopyInp(diopiContextHandle_t ctx, diopiConstTensorHandle_t src
         return diopiSuccess;
     }
 
+    // if src is not dense, change it to preserved-format dense
+    if (!denseCheck(srcTr)) {
+        DiopiTensor denseOut;
+        toDense(ctx, srcTr, denseOut);
+        srcTr = denseOut;
+    }
+
     // memory format convert if memory format is matched.
     // cnnTranspose doesn't support float64 and scalar and contiguousOut only support convertion between the contiguous tensor and the no-contiguous tensor.
-    if (srcTr.shape() == destTr.shape() && srcTr.dim() != 0 && srcTr.dtype() != diopi_dtype_float64 && denseCheck(srcTr) && denseCheck(destTr) &&
-        (destTr.isContiguous() || srcTr.isContiguous())) {
+    if (srcTr.shape() == destTr.shape() && srcTr.dim() != 0 && srcTr.dtype() != diopi_dtype_float64 && denseCheck(destTr)) {
         DiopiTensor destTmpTr = destTr;
         if (destTmpTr.dtype() != srcTr.dtype()) {
             destTmpTr = requiresTensor(ctx, destTr.shape(), srcTr.dtype());
         }
-        DIOPI_CALL(contiguousOut(ctx, srcTr, destTmpTr));
+        DIOPI_CALL(permuteCopy(ctx, srcTr, destTmpTr));
+
         if (destTmpTr.dtype() != destTr.dtype()) {
             DIOPI_CALL(dataTypeCast(ctx, destTr, destTmpTr));
         }
         return diopiSuccess;
     }
-
     // Ordinary copy
     // broadcast
     if (srcTr.shape() != destTr.shape()) {