合并分支 'xiaotong-working' 到 'master'

Xiaotong working

查看合并请求 !5

NiuTrans.Tensor/core/Permute.cpp

+

NiuTrans.Tensor/core/Permute.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2018, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (email: xiaotong@mail.neu.edu.cn) 2018-07-05
+ */
+
+#ifndef __PERMUTE_H__
+#define __PERMUTE_H__
+
+#include "../XTensor.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+    
+/* permute the tensor dimensions on site: a = permuted(a) */
+void Permute_(XTensor * a, int * dimPermute);
+
+/* generate the tensor with permuted dimensions: b = permuted(a) */
+void Permute(XTensor * a, XTensor * b, int * dimPermute);
+    
+} // namespace nts(NiuTrans.Tensor)
+
+#endif // __PERMUTE_H__
+

NiuTrans.Tensor/core/Transpose.cpp

+

NiuTrans.Tensor/core/Transpose.h

+/* NiuTrans.Tensor - an open-source tensor library
+ * Copyright (C) 2017, Natural Language Processing Lab, Northestern University.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * $Created by: XIAO Tong (email: xiaotong@mail.neu.edu.cn) 2018-07-05
+ * It will rain tomorrow - end of the hot days :)
+ */
+
+#ifndef __TRANSPOSE_H__
+#define __TRANSPOSE_H__
+
+#include "../XTensor.h"
+
+namespace nts { // namespace nts(NiuTrans.Tensor)
+
+/* transpose a 1D/2D tensor on site: a = transposed(a) */
+void Transpose_(XTensor * a);
+
+/* generate a transposed 1D/2D tensor: b = transposed(a) */
+void Transpose(XTensor * a, XTensor * b);
+    
+} // namespace nts(NiuTrans.Tensor)
+
+#endif // __TRANSPOSE_H__

source/XLink.cpp

@@ -164,14 +164,42 @@ void XLink::MakeLink(XTensor * t1, XTensor * t2, XTensor * h, const char * typeN
     /* backward for t1 */
     if(t1 != NULL){
         XLink &outgo = t1->outgo;
-        CheckNTErrors(outgo.head != h, "Wrong head of the hyperedge!");
+        CheckNTErrors(outgo.head != t1, "Wrong head of the hyperedge!");
         outgo.AddTail(h);
     }
 
     /* backward for t2 */
     if(t2 != NULL){
         XLink &outgo = t2->outgo;
-        CheckNTErrors(outgo.head != h, "Wrong head of the hyperedge!");
+        CheckNTErrors(outgo.head != t2, "Wrong head of the hyperedge!");
+        outgo.AddTail(h);
+    }
+}
+
+/* 
+create a hyper edge with a list of tensors and a output tensor 
+>> list - a list of input tensors
+>> h - head tensor
+>> typeName - name of edge type
+*/
+void XLink::MakeLink(XList * list, XTensor * h, const char * typeName)
+{
+    /* forward */
+    XLink &income = h->income;
+    income.Reset();
+    income.SetHead(h);
+    income.SetType(typeName);
+
+    for(int i = 0; i < list->count; i++){
+        XTensor * t = (XTensor*)list->GetItem(i);
+        income.AddTail(t);
+    }
+
+    /* backward */
+    for(int i = 0; i < list->count; i++){
+        XTensor * t = (XTensor*)list->GetItem(i);
+        XLink &outgo = t->outgo;
+        CheckNTErrors(outgo.head != t, "Wrong head of the hyperedge!");
         outgo.AddTail(h);
     }
 }

source/XLink.h

@@ -105,6 +105,10 @@ struct XLink
     static
     void MakeLink(XTensor * t1, XTensor * t2, XTensor * h, const char * typeName);
 
+    /* create a hyper edge with a list of tensors and a output tensor */
+    static
+    void MakeLink(XList * list, XTensor * h, const char * typeName);
+
     /* add a parameter */
     static
     void AddParamToHead(XTensor * h, DTYPE param);

source/XName.h

@@ -20,7 +20,7 @@
  * We define various names here
  *
  * $Created by: XIAO Tong (xiaotong@mail.neu.edu.cn) 2018-07-05
- * It was really HOT these days. I can't imagine what a hot day in Shenyang!
+ * It was really HOT these days. I can't imagine what a hot day here in Shenyang!
 */
 
 #ifndef __XNAME_H__
@@ -29,6 +29,13 @@
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
 #define MATH_MATMUL "M_MATMUL"
+#define MATH_CONCATENATESOLY "M_CONCATENATESOLY"
+#define MATH_COPYVALUES "M_COPYVALUES"
+#define MATH_MATRIXMUL "M_MATRIXMUL"
+#define MATH_MATRIXMUL2D "M_MATRIXMUL2D"
+#define MATH_MATRIXMULBATCHED "M_MATRIXMULBATCHED"
+#define MATH_MERGE "M_MERGE"
+#define MATH_MULTIPLY "M_MULTIPLY"
 #define MATH_REDUCEMAX "M_REDUCEMAX"
 #define MATH_REDUCESUM "M_REDUCESUM"
 #define MATH_SELECTRANGE "M_SELECTRANGE"

source/XUtility.cpp

-/* NiuTrans.Tensor - an open-source tensor library
+/* NiuTrans.Tensor - an open-source tensor library
  * Copyright (C) 2017, Natural Language Processing Lab, Northestern University. 
  * All rights reserved.
  *
@@ -403,7 +403,7 @@ int ToCPUInt(int devID, void * value)
     }
 }
 
-/* set the value that is kept on a device */
+/* assign a number to a variable that is kept on a specified device */
 bool SetToDevice(int devID, void * p, DTYPE value)
 {
     if(p == NULL)
@@ -412,7 +412,7 @@ bool SetToDevice(int devID, void * p, DTYPE value)
     if(devID < 0)
         *(DTYPE*)p = value;
     else{
-        XMemCopy(p, devID, &value, -1, sizeof(DTYPE*));
+        XMemCopy(p, devID, &value, -1, sizeof(DTYPE));
     }
 
     return true;

source/core/ConcatenateSolely.cpp

@@ -21,6 +21,7 @@
 
 #include "../XTensor.h"
 #include "../XUtility.h"
+#include "../XName.h"
 #include "ConcatenateSolely.h"
 #include "MergeBlockLists.h"
 
@@ -36,6 +37,10 @@ void ConcatenateSolely(XList * smalls, XTensor * big, int dim)
 {
     CheckNTErrors((big->order > dim && dim >= 0), "Illegal dimension to concatenate!");
 
+    /* make tensor connections */
+    XLink::MakeLink(smalls, big, MATH_CONCATENATESOLY);
+    XLink::AddParamToHeadInt(big, dim);
+
     int catDimSize = 0;
     int dimRDI = big->order - dim - 1;
 

source/core/CopyBlocksInGrid.cpp

@@ -39,7 +39,7 @@ Note that a grid may have a number of blocks
 >> isIndexOnDev - indicates whether the index is on the device already
 */
 void CopyBlocksInGrid(void * source, int blockSize, int blockNum, int gridNum, void * target,
-    int * index, int unitSize, bool isIndexOnDev, XMem * myMem)
+                      int * index, int unitSize, bool isIndexOnDev, XMem * myMem)
 {
     CheckNTErrors((unitSize == sizeof(int)), "TODO!");
 

source/core/CopyValues.cpp

@@ -19,6 +19,7 @@
 * $Created by: XIAO Tong (email: xiaotong@mail.neu.edu.cn) 2018-04-24
 */
 
+#include "../XName.h"
 #include "CopyValues.h"
 #include "CopyValues.cuh"
 
@@ -41,6 +42,9 @@ bool CopyValues(XTensor * s, XTensor * t, XStream * stream)
     CheckNTErrors((t->data != NULL), "Cannot copy to an empty data array!");
     CheckNTErrors((s->unitNum == t->unitNum), "Unmatched data item number!");
 
+    /* make tensor connections */
+    XLink::MakeLink(s, NULL, t, MATH_COPYVALUES);
+
     if ((s->dataType == X_FLOAT16 && t->dataType == X_FLOAT) ||
         (s->dataType == X_FLOAT && t->dataType == X_FLOAT16)) {
         CheckNTErrors(((s->devID < 0 && t->devID < 0) || s->devID == t->devID),

source/core/MakeMergeBlockIndex.cpp

@@ -36,7 +36,7 @@ set target data block index for the data movement in merge
 >> mem - the memory pool
 */
 void MakeMergeBlockIndex(int * blockIndex, int blockNum, int blockNumInMerge,
-    int splitSizeInGrid, int gridSize, int gridNum, XMem * mem)
+                         int splitSizeInGrid, int gridSize, int gridNum, XMem * mem)
 {
     if (mem != NULL && mem->devID >= 0) {
 #ifdef USE_CUDA

source/core/MatrixMULBatchedCPU.cpp

@@ -38,8 +38,8 @@ c_i = trans(a_i) * trans(b_i) * \alpha + c_i * \beta for each i in [0,count-1]
 >> c - output matrix (2d tensor)
 */
 void MatrixMULBatchedCPU(XList * a, MATRIX_TRANS_TYPE transposedA,
-    XList * b, MATRIX_TRANS_TYPE transposedB,
-    XList * c, DTYPE alpha, DTYPE beta)
+                         XList * b, MATRIX_TRANS_TYPE transposedB,
+                         XList * c, DTYPE alpha, DTYPE beta)
 {
     CheckNTErrors((a && b && c), "Empty input lists!");
     CheckNTErrors((a->count == b->count && a->count == c->count), "Input lists must be of the same size!");

source/core/MatrixMul.cpp

@@ -21,6 +21,7 @@
 
 #include "../XTensor.h"
 #include "../XDevice.h"
+#include "../XName.h"
 #include "MatrixMul.h"
 #include "MatrixMul2D.h"
 #include "MatrixMULBatchedCPU.h"
@@ -54,9 +55,16 @@ void MatrixMul(XTensor * a, MATRIX_TRANS_TYPE transposedA,
 {
     CheckNTErrors((a && b && c), "Empty input tensors!");
     CheckNTErrors((a->dataType == b->dataType && a->dataType == c->dataType),
-        "Input tensors should have the same data type!");
+                  "Input tensors should have the same data type!");
     CheckNTErrors((a->order >= 2 && b->order >= 2 && c->order >= 2),
-        "Input tensors must have a order > 2!");
+                  "Input tensors must have a order > 2!");
+
+    /* make tensor connections */
+    XLink::MakeLink(a, b, c, MATH_MATRIXMUL);
+    XLink::AddParamToHeadInt(c, transposedA);
+    XLink::AddParamToHeadInt(c, transposedB);
+    XLink::AddParamToHead(c, alpha);
+    XLink::AddParamToHead(c, beta);
 
     int an = transposedA == X_TRANS ? a->dimSize[1] : a->dimSize[0];
     int am = transposedA == X_TRANS ? a->dimSize[0] : a->dimSize[1];

source/core/MatrixMul2D.cpp

@@ -20,6 +20,7 @@
 */
 
 #include "../XTensor.h"
+#include "../XName.h"
 #include "MatrixMul2D.h"
 #include "MatrixMul2D.cuh"
 #include "MatrixMul2DParallel.h"
@@ -51,6 +52,13 @@ void MatrixMul2D(XTensor * a, MATRIX_TRANS_TYPE transposedA,
     CheckNTErrors((a->order == 2 && b->order == 2 && c->order == 2),
                   "Input tensors must have a order = 2!");
 
+    /* make tensor connections */
+    XLink::MakeLink(a, b, c, MATH_MATRIXMUL2D);
+    XLink::AddParamToHeadInt(c, transposedA);
+    XLink::AddParamToHeadInt(c, transposedB);
+    XLink::AddParamToHead(c, alpha);
+    XLink::AddParamToHead(c, beta);
+
 	int an = a->dimSize[0], am = a->dimSize[1];
 	int bn = b->dimSize[0], bm = b->dimSize[1];
 	int cn = c->dimSize[0], cm = c->dimSize[1];

source/core/MatrixMulBatched.cpp

@@ -21,6 +21,7 @@
 
 #include "../XTensor.h"
 #include "../XDevice.h"
+#include "../XName.h"
 #include "MatrixMulBatched.h"
 #include "MatrixMULBatchedCPU.h"
 #include "XTensorBLAS.h"
@@ -42,15 +43,22 @@ where trans() returns the transposed matrix if the flag is fired
 >> beta - another coefficient
 */
 void MatrixMulBatched(XTensor * a, MATRIX_TRANS_TYPE transposedA,
-    XTensor * b, MATRIX_TRANS_TYPE transposedB,
-    XTensor * c, DTYPE alpha, DTYPE beta,
-    XPRunner * parallelRunner)
+                      XTensor * b, MATRIX_TRANS_TYPE transposedB,
+                      XTensor * c, DTYPE alpha, DTYPE beta,
+                      XPRunner * parallelRunner)
 {
     CheckNTErrors((a && b && c), "Empty input tensors!");
     CheckNTErrors((a->dataType == b->dataType && a->dataType == c->dataType),
-        "Input tensors should have the same data type!");
+                  "Input tensors should have the same data type!");
     CheckNTErrors((a->order >= 2 && b->order >= 2 && c->order >= 2),
-        "Input tensors must have a order > 2!");
+                  "Input tensors must have a order > 2!");
+
+    /* make tensor connections */
+    XLink::MakeLink(a, b, c, MATH_MATRIXMULBATCHED);
+    XLink::AddParamToHeadInt(c, transposedA);
+    XLink::AddParamToHeadInt(c, transposedB);
+    XLink::AddParamToHead(c, alpha);
+    XLink::AddParamToHead(c, beta);
 
     int an = transposedA == X_TRANS ? a->dimSize[1] : a->dimSize[0];
     int am = transposedA == X_TRANS ? a->dimSize[0] : a->dimSize[1];

source/core/Merge.cpp

@@ -21,6 +21,7 @@
 
 #include "../XTensor.h"
 #include "../XUtility.h"
+#include "../XName.h"
 #include "Merge.h"
 #include "MakeMergeBlockIndex.h"
 #include "CopyBlocksOnSite.h"
@@ -63,6 +64,11 @@ void Merge(XTensor * s, XTensor * t, int whereToMerge, int leadingDim)
         }
     }
 
+    /* make tensor connections */
+    XLink::MakeLink(s, NULL, t, MATH_MERGE);
+    XLink::AddParamToHeadInt(t, whereToMerge);
+    XLink::AddParamToHeadInt(t, leadingDim);
+
     int blockSize = 1;
     int blockNum = 1;
     int gridSize = 1;

source/core/MergeBlockLists.cpp

@@ -27,12 +27,12 @@
 namespace nts { // namespace nts(NiuTrans.Tensor)
 
 /*
-    merge data by blocks
-    >> sourceList - list of source data array
-    >> blockSizes - list of the block size for each source data array
-    >> blockNum - number of blocks kept in each data array
-    >> target - target data array
-    >> myMem - memory pool
+merge data by blocks
+>> sourceList - list of source data array
+>> blockSizes - list of the block size for each source data array
+>> blockNum - number of blocks kept in each data array
+>> target - target data array
+>> myMem - memory pool
 */
 void MergeBlockLists(XList * sourceList, int * blockSizes, int blockNum, void * target, XMem * myMem)
 {

source/core/Multiply.cpp

@@ -20,6 +20,7 @@
 */
 
 #include "../XTensor.h"
+#include "../XName.h"
 #include "Multiply.h"
 #include "Multiply.cuh"
 
@@ -41,6 +42,11 @@ void Multiply(XTensor * a, XTensor * b, XTensor * c, int leadingDim, DTYPE alpha
         "Unmatched tensors in multiplication!");
     CheckNTErrors((a->order == b->order && a->order == c->order), "Unmatched tensors!");
 
+    /* make tensor connections */
+    XLink::MakeLink(a, b, c, MATH_MULTIPLY);
+    XLink::AddParamToHeadInt(c, leadingDim);
+    XLink::AddParamToHead(c, alpha);
+
 #ifdef USE_CUDA
     if (a->devID >= 0 || b->devID >= 0 || c->devID >= 0) {
         CudaMultiply(a, b, c, leadingDim, alpha);