use vector buffer to accelerate reduce operation

source/tensor/core/reduce/ReduceMax.cpp

@@ -21,6 +21,8 @@
 
 #include "../../XTensor.h"
 #include "../../XName.h"
+#include "../../XBLAS.h"
+#include "VectorBuffer.h"
 #include "ReduceMax.h"
 #include "ReduceMax.cuh"
 
@@ -76,18 +78,80 @@ void _ReduceMax(const XTensor * input, XTensor * output, int dim)
         }
         blockSize = stride * strideNum;
 
-        for(int k = 0; k < blockNum; k++){
-            DTYPE * ip = (DTYPE*)input->data + blockSize * k;
-            DTYPE * op = (DTYPE*)output->data + stride * k;
-            for(int i = 0; i < stride; i++){
-                DTYPE max = FLOAT_MIN;
-                DTYPE * ipe = ip + blockSize;
-                for(DTYPE * ipb = ip + i; ipb < ipe; ipb += stride){
-                    DTYPE v = *ipb;
-                    if(max < v)
-                        max = v;
+
+        if(input->dimSizeRDI[0] % (4 * 32 / sizeof(DTYPE)) == 0 && input->dimSizeRDI[0] >= 32){
+            int vecBufLength =  32 / sizeof(DTYPE);
+
+            if(dimRDI == 0){
+                //data is contiguous in dim 0
+                for(int i = 0; i < blockNum; i++){
+                    DTYPE * ip = (DTYPE*)input->data + blockSize * i;
+                    DTYPE * op = (DTYPE*)output->data + i;
+                    VectorBuffer vecBuf[4];
+                    for(int j = 0; j < 4; j++){
+                        vecBuf[j] = VectorBuffer::loadu((DTYPE*)(ip) + j * vecBufLength);
+                    }
+                    for(int j = 1; j < strideNum / 32; j++){
+                        const DTYPE* ptr = (DTYPE*)(ip + j * vecBufLength);
+                        vecBuf[0] = vecBuf[0].maxData(VectorBuffer::loadu(ptr + 0 * vecBufLength));
+                        vecBuf[1] = vecBuf[1].maxData(VectorBuffer::loadu(ptr + 1 * vecBufLength));
+                        vecBuf[2] = vecBuf[2].maxData(VectorBuffer::loadu(ptr + 2 * vecBufLength));
+                        vecBuf[3] = vecBuf[3].maxData(VectorBuffer::loadu(ptr + 3 * vecBufLength));
+                    }
+                    vecBuf[0] = vecBuf[0].maxData(vecBuf[1]);
+                    vecBuf[0] = vecBuf[0].maxData(vecBuf[2]);
+                    vecBuf[0] = vecBuf[0].maxData(vecBuf[3]);
+                    DTYPE maxN = DTYPE_MIN;
+                    for(int k = 0; k < vecBufLength; k++){
+                        maxN = MAX(maxN,vecBuf[0][k]);
+                    }
+                    *op = maxN;
+                }
+
+            } else{
+                //data is separated
+                for(int i = 0; i < blockNum; i++){
+                    for(int j = 0; j < input->dimSizeRDI[0] / 32; j++){
+                        DTYPE * ip = (DTYPE*)input->data + blockSize * i;
+                        DTYPE * op = (DTYPE*)output->data + stride * i;
+                        VectorBuffer vecBuf[4];
+                        for(int k = 0; k < 4; k++){
+                            vecBuf[k] = VectorBuffer::loadu((DTYPE*)(ip) + (j * 4 + k) * 32 / sizeof(DTYPE));
+
+                        }
+                        for(int k = 1; k < strideNum; k++){
+                            DTYPE * ptr = ip + k * stride + (j * 4) * vecBufLength;
+                            vecBuf[0] = vecBuf[0].maxData(VectorBuffer::loadu(ptr + 0 * vecBufLength));
+                            vecBuf[1] = vecBuf[1].maxData(VectorBuffer::loadu(ptr + 1 * vecBufLength));
+                            vecBuf[2] = vecBuf[2].maxData(VectorBuffer::loadu(ptr + 2 * vecBufLength));
+                            vecBuf[3] = vecBuf[3].maxData(VectorBuffer::loadu(ptr + 3 * vecBufLength));
+                        }
+                        for(int k = 0; k < 4; k++){
+                            for(int l = 0; l < vecBufLength; l++)
+                                *(op + j * 32 + 8 * k + l) = vecBuf[k][l];
+                        }
+                    }
+                }
+            }
+        }//run vector buffer
+        else{
+            for(int k = 0; k < blockNum; k++){
+                DTYPE * ip = (DTYPE*)input->data + blockSize * k;
+                DTYPE * op = (DTYPE*)output->data + stride * k;
+                for(int i = 0; i < stride; i++){
+    //#if defined(USE_BLAS)
+    //                    *(op + i) = *(ip + i + (int)(stride * IAMAX(strideNum, ip + i, stride)));
+    //#else
+                        DTYPE max = DTYPE_MIN;
+                        DTYPE * ipe = ip + blockSize;
+                        for(DTYPE * ipb = ip + i; ipb < ipe; ipb += stride){
+                            DTYPE v = *ipb;
+                            if(max < v)
+                                max = v;
+                        }
+                        *(op + i) = max;
+    //#endif
                 }
-                *(op + i) = max;
             }
         }
     }

source/tensor/core/reduce/VectorBuffer.h

+#include <cstring>
+#include <cmath>
+#include "../../XGlobal.h"
+
+namespace nts {
+class VectorBuffer {
+private:
+    DTYPE values[32 / sizeof(DTYPE)] = { 0 };
+public:
+    static int size() {
+        return 32 / sizeof(DTYPE);
+    }
+    VectorBuffer() {}
+    VectorBuffer(DTYPE val) {
+        for (int i = 0; i != size(); i++) {
+            values[i] = val;
+        }
+    }
+    static VectorBuffer loadu(const DTYPE* ptr, bool isExp = false, DTYPE power = (DTYPE)1.0F, DTYPE* bias = NULL) {
+        int count = 32 / sizeof(DTYPE);
+        VectorBuffer vec;
+        if (isExp) {
+            if (bias == NULL) {
+                if (power == (DTYPE)1.0) {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = (DTYPE)exp(*(ptr + i));
+                    }
+                }
+                else if (power == (DTYPE)2.0) {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = (DTYPE)exp((*(ptr + i)) * (*(ptr + i)));
+                    }
+                }
+                else if (power == (DTYPE)0.5) {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = (DTYPE)exp(sqrt(*(ptr + i)));
+                    }
+                }
+                else {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = (DTYPE)exp(pow(*(ptr + i), power));
+                    }
+                }
+            }/*is bias == NULL*/
+            else {
+                if (power == (DTYPE)1.0) {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = (DTYPE)exp(*(ptr + i) - bias[i]);
+                    }
+                }
+                else if (power == (DTYPE)2.0) {
+                    for (int i = 0; i != count; i++) {
+                        DTYPE value = *(ptr + i) - bias[i];
+                        vec.values[i] = (DTYPE)exp(value * value);
+                    }
+                }
+                else if (power == (DTYPE)0.5) {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = (DTYPE)exp(sqrt(*(ptr + i) - bias[i]));
+                    }
+                }
+                else {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = (DTYPE)exp(pow(*(ptr + i) - bias[i], power));
+                    }
+                }
+            }
+        }//isExp
+        else {
+            if (bias == NULL) {
+                if (power == (DTYPE)1.0) {
+                    memcpy(vec.values, ptr, count * sizeof(DTYPE));
+                }
+                else if (power == (DTYPE)2.0) {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = (*(ptr + i)) * (*(ptr + i));
+                    }
+                }
+                else if (power == (DTYPE)0.5) {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = (DTYPE)sqrt(*(ptr + i));
+                    }
+                }
+                else {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = (DTYPE)pow(*(ptr + i), power);
+                    }
+                }
+            }// if bias == NULL
+            else {
+                if (power == (DTYPE)1.0) {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = *(ptr + i) - bias[i];
+                    }
+                }
+                else if (power == (DTYPE)2.0) {
+                    for (int i = 0; i != count; i++) {
+                        DTYPE value = *(ptr + i) - bias[i];
+                        vec.values[i] = value * value;
+                    }
+                }
+                else if (power == (DTYPE)0.5) {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = (DTYPE)sqrt(*(ptr + i) - bias[i]);
+                    }
+                }
+                else {
+                    for (int i = 0; i != count; i++) {
+                        vec.values[i] = (DTYPE)pow(*(ptr + i) - bias[i], power);
+                    }
+                }
+            }
+        }
+        return vec;
+    }
+    const DTYPE& operator[](int idx) const {
+        return values[idx];
+    }
+    inline VectorBuffer operator+(const VectorBuffer &a) {
+        for (int i = 0; i != a.size(); i++) {
+            this->values[i] = a[i] + this->values[i];
+        }
+        return *this;
+    }
+    inline VectorBuffer maxData(const VectorBuffer &a) {
+        for (int i = 0; i != a.size(); i++) {
+            this->values[i] = MAX(a[i], this->values[i]);
+        }
+        return *this;
+    }
+};
+}
\ No newline at end of file